Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    /// Conservative monotonic bound sampled before `command.spawn()`, not when
362    /// identity recording finishes or the spawn generation is published.
363    spawned_at: Instant,
364    spawned_at_ms: u64,
365    spawned_from: PathBuf,
366    launch_spec_digest: LaunchSpecDigest,
367    spawned_file_identity: Option<SpawnedFileIdentity>,
368    process_start_time: Option<u64>,
369    process_identity: Option<ProcessIdentity>,
370    pid: u32,
371    /// This process's entry in the daemon's child roster, released when the
372    /// process is reaped or this handle is dropped.
373    roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377    fn id(&self) -> Option<u32> {
378        Some(self.pid)
379    }
380
381    fn process_identity(&self) -> Option<ProcessIdentity> {
382        self.process_identity
383    }
384
385    async fn wait(&mut self) -> io::Result<ExitStatus> {
386        #[cfg(target_os = "macos")]
387        self.confirm_privacy_exec().await;
388        // The roster entry is NOT released here. A daemon shutdown waits for the
389        // roster to empty and then exits the process, so releasing at the reap
390        // let it exit before the exit handler wrote this child's terminal record
391        // (the stderr drain and snapshot update sit in between), and the
392        // shutdown's own `daemon_shutdown` record was intermittently lost. The
393        // caller releases it after recording the exit (`release_roster`), and
394        // dropping the handle releases it too.
395        let result = self.child.wait().await;
396        #[cfg(target_os = "linux")]
397        if result.is_ok() {
398            if let Some(placement) = self.cgroup_placement.as_ref() {
399                cleanup_reaped_cgroup(placement, &self.module_id).await;
400                // Keep ownership while awaiting kernel population changes: a
401                // drain timeout may cancel this wait and then escalate/reap.
402                self.cgroup_placement = None;
403            }
404        }
405        result
406    }
407
408    #[cfg(target_os = "macos")]
409    async fn confirm_privacy_exec(&mut self) {
410        let Some(pending) = &mut self.privacy_exec else {
411            return;
412        };
413        let result = tokio::time::timeout_at(pending.deadline, async {
414            let mut record = Vec::new();
415            loop {
416                let mut ready = pending.reader.readable().await?;
417                let read = ready.try_io(|reader| {
418                    use std::io::Read;
419                    let mut reader = reader.get_ref();
420                    let mut buffer = [0u8; 256];
421                    reader.read(&mut buffer).map(|count| (count, buffer))
422                });
423                match read {
424                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425                    Ok(Ok((count, buffer))) => {
426                        if record.len() + count > 1024 {
427                            return Err(io::Error::other(
428                                "privacy exec refusal record is too long",
429                            ));
430                        }
431                        record.extend_from_slice(&buffer[..count]);
432                    }
433                    Ok(Err(error)) => return Err(error),
434                    Err(_) => continue,
435                }
436            }
437        })
438        .await;
439        #[cfg(test)]
440        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441            use tokio::io::{AsyncReadExt, AsyncWriteExt};
442            if stream.is_none() {
443                *stream = Some(
444                    tokio::net::TcpStream::connect(address.as_str())
445                        .await
446                        .unwrap(),
447                );
448                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449                stream
450                    .as_mut()
451                    .unwrap()
452                    .write_all(&self.pid.to_ne_bytes())
453                    .await
454                    .unwrap();
455            }
456            let mut reply = [0];
457            stream
458                .as_mut()
459                .unwrap()
460                .read_exact(&mut reply)
461                .await
462                .unwrap();
463            pending.confirmation_pause = None;
464            if &reply == b"F" {
465                Ok(Err(io::Error::other("injected late exec refusal")))
466            } else {
467                assert_eq!(&reply, b"X");
468                result
469            }
470        } else {
471            result
472        };
473        // Keep the reader in self across await: select cancellation must not
474        // discard the handshake or reset its original five-second deadline.
475        let pending = self.privacy_exec.as_ref().expect("pending exec");
476        let cause = match result {
477            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478            Ok(Err(error)) => Some(format!(
479                "privacy identity exec acknowledgement failed: {error}"
480            )),
481            Ok(Ok(record)) if !record.is_empty() => Some(
482                std::str::from_utf8(&record)
483                    .ok()
484                    .and_then(|record| {
485                        record
486                            .trim()
487                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488                    })
489                    .filter(|cause| !cause.is_empty())
490                    .unwrap_or("invalid privacy exec refusal record")
491                    .to_string(),
492            ),
493            Ok(Ok(_)) => match self.child.try_wait() {
494                // Empty EOF is the exec acknowledgement. A real module may exit
495                // immediately, including with a reserved trampoline status; no
496                // image is admitted, and its ordinary exit contract stays intact.
497                Ok(Some(_status)) => None,
498                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499                Ok(None) => {
500                    let image = observe_spawned_image(self.pid);
501                    if let Some(image) = image.filter(|image| {
502                        image.executable.is_some()
503                            && image.executable != pending.trampoline
504                            && (image.executable == pending.expected || pending.script)
505                    }) {
506                        if let Err(error) = self
507                            .roster_guard
508                            .as_ref()
509                            .map(|guard| guard.confirm_executable(image))
510                            .transpose()
511                        {
512                            Some(format!(
513                                "could not durably record confirmed module image: {error}"
514                            ))
515                        } else {
516                            let _ = self.report_ready.set(());
517                            info!(module_id = %pending.module_id, pid = self.pid,
518                                "module spawned with own privacy identity (responsibility disclaimed)");
519                            None
520                        }
521                    } else if image.is_none()
522                        || image.is_some_and(|image| image.executable.is_none())
523                    {
524                        // A process can exit between try_wait and the kernel
525                        // image read. Empty EOF already acknowledged exec, so
526                        // preserve that module's ordinary exit rather than
527                        // mislabel a disappearing image as trampoline refusal.
528                        // Keep pending in self across await so cancellation does
529                        // not discard validation or reset its original deadline.
530                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531                            Ok(Ok(_status)) => None,
532                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534                        }
535                    } else {
536                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537                    }
538                }
539            },
540        };
541        let pending = self.privacy_exec.take().expect("pending exec");
542        // Exec confirmation describes launch success, not roster retirement.
543        // Keep even a reaped child recorded until its terminal history exists.
544        self.exec.0.finish(if cause.is_some() {
545            crate::activity::ExecState::Refused
546        } else {
547            crate::activity::ExecState::CommittedExec
548        });
549        if let Some(cause) = cause {
550            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551            self.spawn_failure = Some(cause);
552            // No image is admitted on failure. Reach the entire fresh process
553            // group, including a module which spawned a helper before refusal.
554            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556            }
557            let _ = self.child.start_kill();
558        }
559    }
560
561    /// Releases this child's daemon-shutdown roster entry once its exit has
562    /// been recorded. The pid is already reaped and free for reuse, so the
563    /// entry must not outlive the record any longer than that.
564    fn release_roster(&mut self) {
565        self.roster_guard = None;
566    }
567
568    /// Kill the child and, where containment is available, its process tree.
569    ///
570    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
571    /// helper process leaked the helper — the Synapse embedding module's CUDA
572    /// worker holds the GPU allocation, so the leak cost VRAM until the next
573    /// restart of something else. Terminating the job reaches grandchildren that
574    /// a tree walk cannot, including one whose parent has already exited and
575    /// been reparented away.
576    ///
577    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
578    /// direct-child kill still decides the outcome, so containment can never
579    /// change whether a module is reported as stopped.
580    fn start_kill(&mut self) -> io::Result<()> {
581        #[cfg(windows)]
582        if let Some(job) = &self.job {
583            if let Err(error) = job.terminate() {
584                debug!(
585                    error = %error,
586                    "job termination failed; the direct-child kill still owns the outcome"
587                );
588            }
589        }
590        #[cfg(target_os = "linux")]
591        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592        self.child.start_kill()
593    }
594
595    async fn drain_stderr(&mut self, module_id: &str) {
596        if let Some(pump) = self.stdout_pump.take() {
597            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598        }
599
600        let Some(pump) = self.stderr_pump.take() else {
601            return;
602        };
603        settle_stderr_pump(
604            module_id,
605            &self.stderr_ring,
606            pump,
607            STDERR_PUMP_DRAIN_TIMEOUT,
608        )
609        .await;
610    }
611}
612
613/// Bound the restart's wait, not the lifetime of the exited child's stdout
614/// reader. Slow pipe reads or capture writes must not turn the bound into lost
615/// output; dropping the handle lets the reader finish at EOF, like stderr.
616async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617    match timeout(bound, &mut pump).await {
618        Ok(Ok(())) => {}
619        Ok(Err(error)) => {
620            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621        }
622        Err(_) => {
623            drop(pump);
624            warn!(
625                module_id,
626                waited = ?bound,
627                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628            );
629        }
630    }
631}
632
633/// The reader task for one process's stderr, with the ring generation its
634/// lines are attributed to.
635struct StderrPump {
636    task: JoinHandle<()>,
637    generation: u64,
638}
639
640/// Retire an exited process's stderr reader and wait up to `bound` for it to
641/// reach EOF. A reader still running at the bound is detached, not stopped: it
642/// keeps filling the exited process's section of the ring until its pipe
643/// closes, and the tail reads `Incomplete` until then. See
644/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
645async fn settle_stderr_pump(
646    module_id: &str,
647    ring: &Arc<Mutex<StderrRing>>,
648    pump: StderrPump,
649    bound: Duration,
650) {
651    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652    let StderrPump {
653        mut task,
654        generation,
655    } = pump;
656    lock().retire_pump(generation);
657    match timeout(bound, &mut task).await {
658        Ok(Ok(())) => {}
659        Ok(Err(err)) => {
660            let mut ring = lock();
661            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662            ring.finish_pump(generation);
663            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664        }
665        Err(_) => {
666            // Dropping the handle detaches the task; it ends at EOF on its pipe.
667            drop(task);
668            lock().mark_pump_late(
669                generation,
670                format!(
671                    "stderr of the exited process had not reached EOF {bound:?} after it was \
672                     retired (a descendant may still hold the pipe open); lines it still \
673                     writes are kept in that process's section"
674                ),
675            );
676            warn!(
677                module_id,
678                waited = ?bound,
679                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680            );
681        }
682    }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687    EVENTS.get_or_init(|| {
688        let (sender, _receiver) = watch::channel(0);
689        sender
690    })
691}
692
693pub(crate) fn notify_registration_release() {
694    let events = registration_release_events();
695    let next_generation = (*events.borrow()).wrapping_add(1);
696    events.send_replace(next_generation);
697}
698
699/// How to launch one singleton module process.
700#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702    pub module_id: String,
703    pub program: PathBuf,
704    pub args: Vec<String>,
705    pub env: Vec<(String, String)>,
706    /// When true this is a reserved module: each spawn gets a fresh one-time launch
707    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
708    /// process can register this module_id (a security-boundary module like the
709    /// credential vault must not be impersonable while it is down/restarting).
710    pub reserved: bool,
711    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
712    /// Prefixes come from daemon config and must end in `:` before they reach the
713    /// supervisor; the owner module's current spawn nonce authorizes claims under
714    /// each prefix.
715    pub reserved_prefixes: Vec<String>,
716    /// The wire protocol this module speaks, as DECLARED in daemon config.
717    ///
718    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
719    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
720    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
721    /// and NO launch nonce, and a clean exit the daemon did not request is
722    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
723    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
724    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
725    /// because a process ignores an environment variable it does not read.
726    ///
727    /// The argument is the part that cannot be "harmless to a process that
728    /// ignores it": a stock binary exits on an unknown flag before it listens
729    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
730    /// first conformance run against this mode found it. The nonce is withheld
731    /// because a process that will never present it gains nothing from holding
732    /// it, and a secret in the environment of a process that does not need it is
733    /// a leak surface for no benefit.
734    pub protocol: ModuleProtocol,
735    /// Whether two processes of this module may run at once, which is what a
736    /// blue/green swap does for the length of its overlap. Declared in daemon
737    /// config because the daemon must be able to answer it while the module is
738    /// down, and so a module cannot talk itself into it after registering.
739    pub overlap: ModuleOverlap,
740}
741
742/// Only the configured launch inputs enter this value; spawned connection paths,
743/// nonces and inherited environment never do. Its bytes remain daemon-private.
744#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749        f.write_str("LaunchSpecDigest([redacted])")
750    }
751}
752
753impl LaunchSpecDigest {
754    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755        fn field(hash: &mut Sha256, bytes: &[u8]) {
756            hash.update((bytes.len() as u64).to_le_bytes());
757            hash.update(bytes);
758        }
759        let mut hash = Sha256::new();
760        if let Some(program) = spec.program.to_str() {
761            field(&mut hash, b"utf8-program");
762            field(&mut hash, program.as_bytes());
763        } else {
764            // The daemon config is UTF-8, but in-process callers may provide
765            // native paths: lossy conversion would collapse distinct programs.
766            #[cfg(unix)]
767            {
768                use std::os::unix::ffi::OsStrExt;
769                field(&mut hash, b"unix-program");
770                field(&mut hash, spec.program.as_os_str().as_bytes());
771            }
772            #[cfg(windows)]
773            {
774                use std::os::windows::ffi::OsStrExt;
775                field(&mut hash, b"windows-program");
776                let bytes = spec
777                    .program
778                    .as_os_str()
779                    .encode_wide()
780                    .flat_map(u16::to_le_bytes)
781                    .collect::<Vec<_>>();
782                field(&mut hash, &bytes);
783            }
784            #[cfg(not(any(unix, windows)))]
785            field(&mut hash, spec.program.to_string_lossy().as_bytes());
786        }
787        hash.update((spec.args.len() as u64).to_le_bytes());
788        for arg in &spec.args {
789            field(&mut hash, arg.as_bytes());
790        }
791        let mut env = spec.env.iter().collect::<Vec<_>>();
792        env.sort_unstable();
793        hash.update((env.len() as u64).to_le_bytes());
794        for (key, value) in env {
795            field(&mut hash, key.as_bytes());
796            field(&mut hash, value.as_bytes());
797        }
798        field(
799            &mut hash,
800            match spec.protocol {
801                ModuleProtocol::Subc => b"subc",
802                ModuleProtocol::None => b"none",
803            },
804        );
805        Self(hash.finalize().into())
806    }
807}
808
809/// Whether a module tolerates a second process of itself running alongside.
810///
811/// Most modules are single-writer on their store (a WAL, a capture log, a
812/// resident index behind a writer barrier), and two processes on one store
813/// corrupt it. So a swap, which overlaps the old and new process by design,
814/// is refused unless the module's config opts in with `overlap: "safe"`.
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817    /// Never run two processes of this module at once. The default.
818    #[default]
819    Exclusive,
820    /// The module has said a second process of itself is harmless for the
821    /// length of a swap.
822    ///
823    /// Declare it only if a second instance can run for a few seconds without
824    /// touching ANY single-writer store: every database, WAL, index, projector
825    /// and scheduled job the module owns. A lease on part of that state is not
826    /// enough. broca's session lease guards WAL appends while its run index, its
827    /// store projector and its archive fold timer (which unlinks live WAL files)
828    /// stay single-writer, so broca is exclusive despite holding a lease. The
829    /// refusal only fires after this has been decided, so the decision is the
830    /// check.
831    Safe,
832}
833
834impl ModuleOverlap {
835    pub fn as_str(self) -> &'static str {
836        match self {
837            Self::Exclusive => "exclusive",
838            Self::Safe => "safe",
839        }
840    }
841}
842
843/// Environment variable telling a spawned module which case it was started
844/// for, before it sends HELLO. Only a swap candidate carries it, as
845/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
846///
847/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
848/// longer because nobody waits on it, while a plain restart must flip ready
849/// quickly because callers see `module_warming` until it does. Absence means
850/// plain restart, the safe reading. The daemon trusts nothing about it; the
851/// candidate is proven by its launch nonce at HELLO.
852pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
854pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855/// How long a swap waits for its candidate to register and declare itself
856/// ready when the operator does not say. A module warming as a swap candidate
857/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
858/// daemon allows that plus time to start the process and send HELLO.
859pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861/// Bounded restart policy for crash exits.
862///
863/// `max_restarts` is the number of replacement processes allowed after the
864/// initial spawn WITHIN `window`. After that many crash restarts inside one
865/// window the module enters [`ModuleState::Failed`] and the supervisor stops
866/// the crash loop.
867///
868/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
869/// and that only survived because crashes were rare: a module that crashed
870/// three times across a week was disabled forever by crashes that had nothing
871/// to do with each other. That stopped being survivable once modules began
872/// exiting non-zero whenever the daemon's connection to them drops, because
873/// then every daemon-side connection drop spends a unit of the same budget and
874/// one flappy hour permanently stops a healthy module. Restarts older than
875/// `window` release their slot, so a module that crashed twice yesterday has a
876/// full budget today, while a genuine crash loop -- which is fast by
877/// definition -- still reaches the cap and stops.
878#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880    pub max_restarts: u32,
881    /// Base delay before a crash replacement. The actual delay escalates with
882    /// the number of recent crash replacements and is capped by `max_backoff`.
883    pub backoff: Duration,
884    /// Maximum delay before a crash replacement.
885    pub max_backoff: Duration,
886    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
887    /// budget effectively infinite (nothing is ever in-window), which is why
888    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
889    pub window: Duration,
890}
891
892impl RestartPolicy {
893    /// A policy with the default crash window. Callers that care about the
894    /// window say so with [`Self::with_window`]; the ones that do not are
895    /// asking for the standard rate limit, not for no limit.
896    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897        Self {
898            max_restarts,
899            backoff,
900            max_backoff: DEFAULT_MAX_BACKOFF,
901            window: DEFAULT_RESTART_WINDOW,
902        }
903    }
904
905    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906        self.max_backoff = max_backoff;
907        self
908    }
909
910    pub fn with_window(mut self, window: Duration) -> Self {
911        self.window = window;
912        self
913    }
914
915    /// Calculate the capped exponential delay for the next crash replacement.
916    /// `restart_in_window` is zero for the first replacement after an operator
917    /// action (restart, reload, re-enable) cleared the crash ring, or after all
918    /// older crash replacements have aged out of the window.
919    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920        if self.backoff.is_zero() || self.max_backoff.is_zero() {
921            return Duration::ZERO;
922        }
923
924        let mut delay = self.backoff;
925        for _ in 0..restart_in_window {
926            if delay >= self.max_backoff {
927                return self.max_backoff;
928            }
929            delay = delay
930                .checked_mul(10)
931                .unwrap_or(self.max_backoff)
932                .min(self.max_backoff);
933        }
934        delay.min(self.max_backoff)
935    }
936
937    /// The one sentence that explains a budget-exhausted stop, used for both the
938    /// log line and the terminal record so the two cannot drift. It names the
939    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
940    /// exactly what this budget is not.
941    fn budget_exhausted_detail(&self) -> String {
942        format!(
943            "crash budget exhausted: max_restarts={} within window_secs={}",
944            self.max_restarts,
945            self.window.as_secs()
946        )
947    }
948}
949
950impl Default for RestartPolicy {
951    fn default() -> Self {
952        Self {
953            max_restarts: DEFAULT_MAX_RESTARTS,
954            backoff: DEFAULT_BACKOFF,
955            max_backoff: DEFAULT_MAX_BACKOFF,
956            window: DEFAULT_RESTART_WINDOW,
957        }
958    }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963    restart_in_window: u32,
964    delay: Duration,
965}
966
967/// Whether the daemon itself will bring this module back after the exit being
968/// handled: it is enabled AND its in-window crash restarts are below the cap.
969///
970/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
971/// the window are dropped here rather than by a timer, so the count is right
972/// the moment somebody asks and no bookkeeping runs for idle modules.
973fn daemon_will_restart(
974    state: &mut SupervisorSnapshot,
975    policy: &RestartPolicy,
976    now: Instant,
977) -> bool {
978    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988    Report,
989    Restart,
990    Alert,
991}
992
993impl fmt::Display for HealthAction {
994    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995        f.write_str(match self {
996            Self::Report => "report",
997            Self::Restart => "restart",
998            Self::Alert => "alert",
999        })
1000    }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005    /// Optional loopback HTTP endpoint for a managed non-wire process.
1006    /// Changing it applies live on rescan; the process protocol changes only
1007    /// at its next spawn.
1008    pub http: Option<String>,
1009    pub cadence: Duration,
1010    pub deadline: Duration,
1011    pub failure_threshold: u32,
1012    pub on_degraded: HealthAction,
1013    pub on_failing: HealthAction,
1014    pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018    fn default() -> Self {
1019        Self {
1020            http: None,
1021            cadence: DEFAULT_HEALTH_CADENCE,
1022            deadline: DEFAULT_HEALTH_DEADLINE,
1023            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024            on_degraded: HealthAction::Report,
1025            on_failing: HealthAction::Report,
1026            critical: false,
1027        }
1028    }
1029}
1030
1031/// The supervisor's view of one module's health, relayed to clients over
1032/// channel-0 and rendered by `ck health`.
1033///
1034/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1035/// stated here rather than only at the wire type a consumer reads. A reader can
1036/// look up what `None` means; only a writer can silently change it, and the
1037/// writer has no reason to go looking at a downstream contract before editing.
1038///
1039/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1040/// back to `None` on re-registration precisely so a respawned module does not
1041/// carry its predecessor's timestamp — so an old value and an absent one call for
1042/// opposite readings, and anything that defaulted this to a number would make a
1043/// never-probed module indistinguishable from one probed at the epoch.
1044///
1045/// `detail` and `metrics` are `None` when the module published none on this
1046/// probe, which does not mean it reported nothing wrong — it is also the shape
1047/// when the probe never reached it. `last_probe_ms` is what separates those.
1048#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050    pub status: SupervisorHealthStatus,
1051    pub last_probe_ms: Option<u64>,
1052    pub detail: Option<String>,
1053    pub metrics: Option<Value>,
1054    pub consecutive_failures: u32,
1055    /// Number of replies received after a recurring health probe's deadline.
1056    /// Unlike a timeout, every increment proves the module was alive.
1057    pub late_answer_count: u64,
1058    /// End-to-end latency of the newest late reply, measured from probe start.
1059    pub last_late_answer_latency_ms: Option<u64>,
1060    pub last_action: Option<String>,
1061    /// Set together with `last_action`; the pair moves as one, and both being
1062    /// absent means no escalation has ever been taken rather than that the last
1063    /// one succeeded.
1064    pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068    fn default() -> Self {
1069        Self {
1070            status: SupervisorHealthStatus::Unknown,
1071            last_probe_ms: None,
1072            detail: None,
1073            metrics: None,
1074            consecutive_failures: 0,
1075            late_answer_count: 0,
1076            last_late_answer_latency_ms: None,
1077            last_action: None,
1078            last_action_ms: None,
1079        }
1080    }
1081}
1082
1083/// Typed lifecycle state for a supervised module.
1084#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086    Starting,
1087    Running,
1088    Unresponsive,
1089    Restarting,
1090    Draining,
1091    Stopped,
1092    Failed,
1093    Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098        f.write_str(match self {
1099            Self::Starting => "starting",
1100            Self::Running => "running",
1101            Self::Unresponsive => "unresponsive",
1102            Self::Restarting => "restarting",
1103            Self::Draining => "draining",
1104            Self::Stopped => "stopped",
1105            Self::Failed => "failed",
1106            Self::Disabled => "disabled",
1107        })
1108    }
1109}
1110
1111/// Supervisor classification of a child-process exit.
1112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114    Clean,
1115    Crash,
1116    DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120    fn from(kind: ExitKind) -> Self {
1121        match kind {
1122            ExitKind::Clean => Self::Clean,
1123            ExitKind::Crash => Self::Crash,
1124            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125        }
1126    }
1127}
1128
1129/// Exact process identity retained when a supervised module registers its
1130/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133    pub(crate) pid: u32,
1134    pub(crate) start_time: u64,
1135}
1136
1137/// Last observed child exit, if any.
1138#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140    pub kind: ExitKind,
1141    pub code: Option<i32>,
1142    pub signal: Option<i32>,
1143    pub at_ms: u64,
1144}
1145
1146/// Point-in-time module status answerable by subc without forwarding to the
1147/// module process.
1148#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150    pub module_id: String,
1151    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1152    pub waiting_for_previous_pid: Option<u32>,
1153    /// Why the previous instance still prevents a new start.
1154    pub waiting_for_previous_detail: Option<String>,
1155    /// The daemon's Windows per-spawn record: `pipe` after the process it
1156    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1157    /// List/status reports use this record, not the source the module sends
1158    /// in its registration message's manifest.provenance.launch_nonce_source.
1159    pub launch_nonce_source: Option<String>,
1160    pub state: ModuleState,
1161    pub enabled: bool,
1162    pub process_alive: bool,
1163    pub registration_active: bool,
1164    /// The module's declared wire protocol, carried beside `live` because it is
1165    /// what makes `live` readable: the two fields answer one question together.
1166    /// While a process is alive this is its launch declaration, not a later
1167    /// pending-reload edit. When down it is the configured next launch protocol.
1168    pub protocol: ModuleProtocol,
1169    /// Whether the module is serving, under the strongest definition the daemon
1170    /// can assert for its protocol.
1171    ///
1172    /// A subc module must also be REGISTERED: its process being alive says
1173    /// nothing about whether it can take a request. A `protocol: "none"` module
1174    /// never registers, so that term is dropped and this falls back to "enabled,
1175    /// running, and the process the daemon launched is alive" -- which is all
1176    /// the daemon observes about a process that speaks no subc wire. It stays a
1177    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1178    /// rather than printing it bare.
1179    pub live: bool,
1180    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1181    /// restarts have already released their slot, so this count can go down
1182    /// without anybody touching the module.
1183    pub restart_count: u32,
1184    /// Replacement processes spawned over this module's entire supervisor lifetime;
1185    /// unlike `restart_count`, this value is never reset by an operator action
1186    /// and never falls out of a window.
1187    pub lifetime_restarts: u32,
1188    pub spawn_generation: u64,
1189    /// The budget `restart_count` is spent against. Carried alongside the count
1190    /// because the count alone does not say how close the module is to being
1191    /// disabled, and reporting one without the other is what makes an
1192    /// about-to-be-retired module look ordinary.
1193    pub max_restarts: u32,
1194    /// The span `restart_count` is counted over. Carried with the pair above for
1195    /// the same reason they are carried together: "2 of 3" means one thing for a
1196    /// ten-minute window and something else entirely for a lifetime.
1197    pub restart_window: Duration,
1198    /// Effective drain and restart timing policy used by this running module.
1199    /// These values are carried together with the restart budget so status
1200    /// readers can compare configured intent with what the supervisor applied.
1201    pub drain_timeout: Duration,
1202    pub restart_backoff: Duration,
1203    pub restart_max_backoff: Duration,
1204    /// The module's process. On macOS this stays absent while the `ck-subc`
1205    /// launch trampoline is still running in that pid, and appears once the
1206    /// exec acknowledgement confirms the module image has replaced it. Launch
1207    /// time and the supervisor's own hold on the process are unaffected.
1208    pub pid: Option<u32>,
1209    pub spawned_at_ms: Option<u64>,
1210    pub spawned_from: Option<PathBuf>,
1211    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212    pub process_start_time: Option<u64>,
1213    pub last_exit: Option<ExitReport>,
1214    pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220    generation: u64,
1221    turn: u64,
1222    registered_connection: Option<ConnectionId>,
1223    next_probe_at: Option<Instant>,
1224    wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230    #[cfg(windows)]
1231    pipe_consumed: Option<Arc<OnceLock<()>>>,
1232    /// Counts running-child loop turns, not executor polls of a parked wait.
1233    #[cfg(test)]
1234    actor_turns: u64,
1235    /// Running select after exec confirmation and with no unconsumed registry
1236    /// event. Recording a spawn or starting a loop turn is not this barrier.
1237    #[cfg(test)]
1238    actor_select: Option<ActorSelectCheckpoint>,
1239    state: ModuleState,
1240    enabled: bool,
1241    process_alive: bool,
1242    spawned_protocol: Option<ModuleProtocol>,
1243    spawn_failure: Option<String>,
1244    waiting_for_previous_pid: Option<u32>,
1245    waiting_for_previous_detail: Option<String>,
1246    /// When each crash restart was spent, oldest first. This IS the crash
1247    /// budget: its in-window length is the count an operator sees and the count
1248    /// the restart decision is made against, so there is no second counter that
1249    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1250    /// operator actions that used to zero the old lifetime counter.
1251    crash_restarts: VecDeque<Instant>,
1252    lifetime_restarts: u32,
1253    /// Successful child spawns in this daemon incarnation.
1254    ///
1255    /// `lifetime_restarts` was considered and rejected: it starts at zero
1256    /// (line 640), successful initial/operator spawns in `set_running` do not
1257    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1258    /// increments before a successful replacement exists (lines 604, 3846,
1259    /// and 3921), so a failed spawn can consume it. This counter moves only
1260    /// when a live PID is accepted below.
1261    spawn_generation: u64,
1262    pid: Option<u32>,
1263    #[cfg(target_os = "macos")]
1264    report_ready: Option<Arc<OnceLock<()>>>,
1265    /// Last reaped child, retained after current process facts are cleared.
1266    reaped_pid: Option<u32>,
1267    /// Whether a replacement has already been scheduled for the current process.
1268    respawn_pending: bool,
1269    /// A second restart is waiting for the replacement already scheduled.
1270    coalesced_restart_pending: bool,
1271    spawned_at_ms: Option<u64>,
1272    spawned_from: Option<PathBuf>,
1273    launch_spec_digest: Option<LaunchSpecDigest>,
1274    spawned_file_identity: Option<SpawnedFileIdentity>,
1275    process_start_time: Option<u64>,
1276    deliberate_severance: Option<ProcessIdentity>,
1277    last_exit: Option<ExitReport>,
1278    /// Diagnostic attached to the next drain's terminal record, if any.
1279    drain_disposition_detail: Option<String>,
1280    health: ModuleHealthStatus,
1281    /// Whether the current process was started as a swap candidate and so
1282    /// lives in the module's alternate cgroup. The next swap's candidate takes
1283    /// the other one, so the two processes of a swap never share a cgroup. A
1284    /// plain spawn always uses the primary cgroup.
1285    in_alternate_slot: bool,
1286    /// Whether the current `Draining` state ends in a replacement process
1287    /// (restart, reload, health restart) rather than a stop. Only meaningful
1288    /// while `state` is `Draining`; every entry into that state rewrites it.
1289    /// It is what lets route.open answer the retryable `module_reloading` to a
1290    /// consumer that reaches a still-registered process mid-restart, instead of
1291    /// the `supervisor_not_live` a stop or disable deserves.
1292    draining_to_replace: bool,
1293    /// Whether a configuration update has been applied since the current
1294    /// process was spawned, so that process runs an older spec than the one
1295    /// the supervisor now holds. A queued restart is only coalesced into a
1296    /// fresher process when this is false: a restart requested to pick up a
1297    /// new configuration must not be satisfied by a process that predates it.
1298    configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302    /// The pid that status, provenance and resource readings may report. While
1303    /// the launch trampoline still runs in the pid, reading its executable or
1304    /// resource use would describe `ck-subc`, not the module, so none is
1305    /// reported until the exec acknowledgement confirms the module image.
1306    fn reported_pid(&self) -> Option<u32> {
1307        #[cfg(target_os = "macos")]
1308        if self
1309            .report_ready
1310            .as_ref()
1311            .is_some_and(|ready| ready.get().is_none())
1312        {
1313            return None;
1314        }
1315        self.pid
1316    }
1317
1318    fn starting() -> Self {
1319        Self::new(ModuleState::Starting, true)
1320    }
1321
1322    fn disabled() -> Self {
1323        Self::new(ModuleState::Disabled, false)
1324    }
1325
1326    /// Crash restarts still inside `window`, having dropped the ones that are
1327    /// not. Pruning on read is what makes the budget a rate: an instant older
1328    /// than the window stops holding a slot the moment anybody counts.
1329    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330        while let Some(oldest) = self.crash_restarts.front() {
1331            if now.duration_since(*oldest) > window {
1332                self.crash_restarts.pop_front();
1333            } else {
1334                break;
1335            }
1336        }
1337        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338    }
1339
1340    /// Spend one unit of the crash budget and record the restart in the ledger.
1341    ///
1342    /// The ring is bounded by the cap because more than `max_restarts` in-window
1343    /// instants can never be reached (the caller refuses the restart first), so
1344    /// anything beyond that is an unbounded queue waiting to happen.
1345    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346        self.crash_restarts.push_back(now);
1347        while self.crash_restarts.len() > policy.max_restarts as usize {
1348            self.crash_restarts.pop_front();
1349        }
1350        self.lifetime_restarts += 1;
1351    }
1352
1353    /// Reserve one crash-restart slot and calculate the delay before respawning.
1354    /// The count is captured before recording this restart, so the first retry
1355    /// uses the base delay and each later in-window retry escalates once.
1356    fn next_crash_restart(
1357        &mut self,
1358        policy: &RestartPolicy,
1359        now: Instant,
1360    ) -> Option<CrashRestartSchedule> {
1361        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362        if restart_in_window >= policy.max_restarts {
1363            return None;
1364        }
1365        self.record_crash_restart(policy, now);
1366        Some(CrashRestartSchedule {
1367            restart_in_window,
1368            delay: policy.delay_for_restart(restart_in_window),
1369        })
1370    }
1371
1372    /// Give the module its full budget back, as an operator restart, reload, or
1373    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1374    /// ledger of what actually happened, and an operator action does not unmake
1375    /// the crashes.
1376    fn clear_crash_restarts(&mut self) {
1377        self.crash_restarts.clear();
1378    }
1379
1380    fn new(state: ModuleState, enabled: bool) -> Self {
1381        Self {
1382            exec_outcome: None,
1383            #[cfg(test)]
1384            actor_turns: 0,
1385            #[cfg(test)]
1386            actor_select: None,
1387            state,
1388            enabled,
1389            process_alive: false,
1390            #[cfg(windows)]
1391            pipe_consumed: None,
1392            spawned_protocol: None,
1393            spawn_failure: None,
1394            waiting_for_previous_pid: None,
1395            waiting_for_previous_detail: None,
1396            crash_restarts: VecDeque::new(),
1397            lifetime_restarts: 0,
1398            spawn_generation: 0,
1399            pid: None,
1400            #[cfg(target_os = "macos")]
1401            report_ready: None,
1402            reaped_pid: None,
1403            respawn_pending: false,
1404            coalesced_restart_pending: false,
1405            spawned_at_ms: None,
1406            spawned_from: None,
1407            launch_spec_digest: None,
1408            spawned_file_identity: None,
1409            process_start_time: None,
1410            deliberate_severance: None,
1411            last_exit: None,
1412            drain_disposition_detail: None,
1413            health: ModuleHealthStatus::default(),
1414            in_alternate_slot: false,
1415            draining_to_replace: false,
1416            configuration_updated_since_spawn: false,
1417        }
1418    }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427    version: u8,
1428    frames: mpsc::Sender<Frame>,
1429    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1430    /// from which event. The full frame channel cannot carry that news, so it
1431    /// travels beside it; see `SpawnEventFeed::subscribe`.
1432    lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437    daemon_incarnation: String,
1438    seq: u64,
1439    capacity: usize,
1440    live: HashMap<String, LiveSpawn>,
1441    generations: HashMap<String, u64>,
1442    events: VecDeque<SpawnEvent>,
1443    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447    fn default() -> Self {
1448        Self {
1449            daemon_incarnation: "unconfigured".to_string(),
1450            seq: 0,
1451            capacity: SPAWN_EVENT_RING_CAPACITY,
1452            live: HashMap::new(),
1453            generations: HashMap::new(),
1454            events: VecDeque::new(),
1455            subscribers: HashMap::new(),
1456        }
1457    }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462    Arc<Mutex<SpawnEventState>>,
1463    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468    ForeignIncarnation { current: String },
1469    TooOld { oldest: SpawnCursor },
1470    Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474    fn configure_incarnation(&self, daemon_incarnation: String) {
1475        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476        state.daemon_incarnation = daemon_incarnation;
1477        state.seq = 0;
1478        state.live.clear();
1479        state.generations.clear();
1480        state.events.clear();
1481        state.subscribers.clear();
1482    }
1483
1484    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485        SpawnCursor {
1486            daemon_incarnation: state.daemon_incarnation.clone(),
1487            seq: state.seq,
1488        }
1489    }
1490
1491    fn snapshot(&self) -> SpawnSnapshot {
1492        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495        SpawnSnapshot {
1496            cursor: Self::cursor(&state),
1497            ring_bound: state.capacity as u64,
1498            live,
1499        }
1500    }
1501
1502    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504        let generation = state
1505            .generations
1506            .get(module_id)
1507            .copied()
1508            .unwrap_or(0)
1509            .checked_add(1)
1510            .expect("spawn generation exhausted");
1511        state.generations.insert(module_id.to_string(), generation);
1512        let live = LiveSpawn {
1513            module_id: module_id.to_string(),
1514            spawn_generation: generation,
1515            pid,
1516            spawned_at_ms,
1517        };
1518        state.live.insert(module_id.to_string(), live);
1519        Self::emit_locked(
1520            &mut state,
1521            SpawnEventKind::Spawned,
1522            module_id.to_string(),
1523            generation,
1524            pid,
1525            None,
1526            None,
1527        );
1528        generation
1529    }
1530
1531    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533        let Some(live) = state.live.remove(module_id) else {
1534            warn!(
1535                module_id,
1536                "terminal record had no live spawn event identity"
1537            );
1538            return;
1539        };
1540        Self::emit_locked(
1541            &mut state,
1542            SpawnEventKind::Exited,
1543            module_id.to_string(),
1544            live.spawn_generation,
1545            live.pid,
1546            exit_code,
1547            exit_signal,
1548        );
1549    }
1550
1551    /// Report the exit of a process that a swap has already replaced.
1552    ///
1553    /// `emit_exited` removes the module's live entry, which after a swap's
1554    /// cutover describes the promoted candidate, not the old process now
1555    /// exiting. This emits the old generation's exit and leaves the live entry
1556    /// alone unless it still names that generation.
1557    fn emit_superseded_exited(
1558        &self,
1559        module_id: &str,
1560        spawn_generation: u64,
1561        pid: u32,
1562        exit_code: Option<i32>,
1563        exit_signal: Option<i32>,
1564    ) {
1565        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566        if state
1567            .live
1568            .get(module_id)
1569            .is_some_and(|live| live.spawn_generation == spawn_generation)
1570        {
1571            state.live.remove(module_id);
1572        }
1573        Self::emit_locked(
1574            &mut state,
1575            SpawnEventKind::Exited,
1576            module_id.to_string(),
1577            spawn_generation,
1578            pid,
1579            exit_code,
1580            exit_signal,
1581        );
1582    }
1583
1584    #[allow(clippy::too_many_arguments)]
1585    fn emit_locked(
1586        state: &mut SpawnEventState,
1587        kind: SpawnEventKind,
1588        module_id: String,
1589        spawn_generation: u64,
1590        pid: u32,
1591        exit_code: Option<i32>,
1592        exit_signal: Option<i32>,
1593    ) {
1594        state.seq = state
1595            .seq
1596            .checked_add(1)
1597            .expect("spawn event sequence exhausted");
1598        let event = SpawnEvent {
1599            cursor: Self::cursor(state),
1600            kind,
1601            module_id,
1602            spawn_generation,
1603            pid,
1604            exit_code,
1605            exit_signal,
1606        };
1607        state.events.push_back(event.clone());
1608        while state.events.len() > state.capacity {
1609            state.events.pop_front();
1610        }
1611        let body = match serde_json::to_vec(&event) {
1612            Ok(body) => body,
1613            Err(error) => {
1614                error!(%error, "failed to serialize supervisor spawn event");
1615                return;
1616            }
1617        };
1618        state.subscribers.retain(|(connection_id, corr), subscriber| {
1619            let frame = Frame::build_with_version(
1620                subscriber.version,
1621                FrameType::StreamData,
1622                control_flags(),
1623                0,
1624                0,
1625                *corr,
1626                body.clone(),
1627            );
1628            match frame {
1629                Ok(frame) => {
1630                    if subscriber.frames.try_send(frame).is_ok() {
1631                        true
1632                    } else {
1633                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634                        if let Some(lagged) = subscriber.lagged.take() {
1635                            let _ = lagged.send(event.cursor.clone());
1636                        }
1637                        false
1638                    }
1639                }
1640                Err(error) => {
1641                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642                    false
1643                }
1644            }
1645        });
1646    }
1647
1648    fn subscribe(
1649        &self,
1650        connection_id: ConnectionId,
1651        corr: u64,
1652        version: u8,
1653        since: Option<SpawnCursor>,
1654        sink: FrameSink,
1655    ) -> Result<(), SpawnSubscribeRefusal> {
1656        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658        {
1659            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660            let replay = if let Some(since) = since {
1661                if since.daemon_incarnation != state.daemon_incarnation {
1662                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663                        current: state.daemon_incarnation.clone(),
1664                    });
1665                }
1666                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667                    if since.seq < oldest.seq.saturating_sub(1) {
1668                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669                    }
1670                }
1671                state
1672                    .events
1673                    .iter()
1674                    .filter(|event| event.cursor.seq > since.seq)
1675                    .cloned()
1676                    .collect::<Vec<_>>()
1677            } else {
1678                Vec::new()
1679            };
1680            for event in replay {
1681                let body = serde_json::to_vec(&event)
1682                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683                let frame = Frame::build_with_version(
1684                    version,
1685                    FrameType::StreamData,
1686                    control_flags(),
1687                    0,
1688                    0,
1689                    corr,
1690                    body,
1691                )
1692                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693                frames
1694                    .try_send(frame)
1695                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696            }
1697            state.subscribers.insert(
1698                (connection_id, corr),
1699                SpawnSubscriber {
1700                    version,
1701                    frames,
1702                    lagged: Some(lagged),
1703                },
1704            );
1705        }
1706        // The lagged terminal is sent here, by the forwarder, rather than by
1707        // the emitter: at the moment of the drop the subscriber's own channel
1708        // is full, and writing to the connection sink directly from the emitter
1709        // would put the Error AHEAD of the events still queued in that channel
1710        // (and the emitter holds the feed lock, so it cannot await the sink).
1711        // Dropping the subscriber drops the only sender, so `recv` drains every
1712        // queued event and then returns `None`; only then is the Error sent, so
1713        // the client sees each event it can keep, then the reason it was cut.
1714        // Cancel and connection removal drop the oneshot unsent, so they end
1715        // the stream with no Error.
1716        tokio::spawn(async move {
1717            while let Some(frame) = receiver.recv().await {
1718                if sink.send(frame).await.is_err() {
1719                    return;
1720                }
1721            }
1722            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723                return;
1724            };
1725            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726                Ok(frame) => {
1727                    let _ = sink.send(frame).await;
1728                }
1729                Err(error) => {
1730                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731                }
1732            }
1733        });
1734        Ok(())
1735    }
1736
1737    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738        let Some(subscriber) = self
1739            .0
1740            .lock()
1741            .unwrap_or_else(|p| p.into_inner())
1742            .subscribers
1743            .remove(&(connection_id, corr))
1744        else {
1745            return false;
1746        };
1747        if let Ok(frame) = Frame::build_with_version(
1748            subscriber.version,
1749            FrameType::StreamEnd,
1750            control_flags(),
1751            0,
1752            0,
1753            corr,
1754            Vec::new(),
1755        ) {
1756            tokio::spawn(async move {
1757                let _ = subscriber.frames.send(frame).await;
1758            });
1759        }
1760        true
1761    }
1762
1763    fn remove_connection(&self, connection_id: ConnectionId) {
1764        self.0
1765            .lock()
1766            .unwrap_or_else(|p| p.into_inner())
1767            .subscribers
1768            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769    }
1770
1771    #[cfg(any(test, feature = "test-support"))]
1772    fn set_capacity(&self, capacity: usize) {
1773        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774    }
1775
1776    #[cfg(any(test, feature = "test-support"))]
1777    fn subscriber_count(&self) -> usize {
1778        self.0
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .subscribers
1782            .len()
1783    }
1784}
1785
1786/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1787/// The terminal Error a lagged spawn subscriber receives after its queued events.
1788fn spawn_subscriber_lagged_frame(
1789    version: u8,
1790    corr: u64,
1791    first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796            .to_string(),
1797        detail: Some(serde_json::json!({
1798            "first_undelivered_cursor": first_undelivered
1799        })),
1800    })
1801    .map_err(|error| error.to_string())?;
1802    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803        .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807    fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809    /// Whether the supervisor is replacing this module's process right now: an
1810    /// operator restart or reload, a health restart, or a crash respawn whose
1811    /// backoff is running. A module in that state is not live, but a consumer
1812    /// refused now should retry shortly rather than treat the target as gone.
1813    /// Stopped, failed, and disabled modules are not replacing.
1814    fn process_replacing(&self, _module_id: &str) -> bool {
1815        false
1816    }
1817}
1818
1819/// Shared process-liveness registry keyed by supervised `module_id`.
1820#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826    pub fn new() -> Self {
1827        Self::default()
1828    }
1829
1830    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831        let mut snapshots = self
1832            .snapshots
1833            .lock()
1834            .unwrap_or_else(|poisoned| poisoned.into_inner());
1835        snapshots.insert(module_id, snapshot);
1836    }
1837
1838    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        let is_current = snapshots
1844            .get(module_id)
1845            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846            .unwrap_or(false);
1847        if is_current {
1848            snapshots.remove(module_id);
1849        }
1850    }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854    fn process_live(&self, module_id: &str) -> Option<bool> {
1855        let snapshot = {
1856            let snapshots = self
1857                .snapshots
1858                .lock()
1859                .unwrap_or_else(|poisoned| poisoned.into_inner());
1860            snapshots.get(module_id).cloned()
1861        }?;
1862        let snapshot = snapshot
1863            .lock()
1864            .unwrap_or_else(|poisoned| poisoned.into_inner());
1865        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866    }
1867
1868    fn process_replacing(&self, module_id: &str) -> bool {
1869        let Some(snapshot) = self
1870            .snapshots
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner())
1873            .get(module_id)
1874            .cloned()
1875        else {
1876            return false;
1877        };
1878        let snapshot = snapshot
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner());
1881        snapshot.enabled
1882            && match snapshot.state {
1883                ModuleState::Restarting => true,
1884                ModuleState::Draining => snapshot.draining_to_replace,
1885                ModuleState::Starting
1886                | ModuleState::Running
1887                | ModuleState::Unresponsive
1888                | ModuleState::Stopped
1889                | ModuleState::Failed
1890                | ModuleState::Disabled => false,
1891            }
1892    }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898    reached: tokio::sync::Notify,
1899    resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904    Spawn,
1905    Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910    deadline: Instant,
1911    kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918    start_requester: Option<Principal>,
1919    pending_activity_restart: bool,
1920    provenance_probe: ExecutableIdentityProbe,
1921    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1922    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923    /// A reload acknowledges completion only after its replacement registers.
1924    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925    restart_policy: RestartPolicy,
1926    /// This module's RESOLVED drain budget: per-module config when present,
1927    /// else `default_drain_timeout`.
1928    drain_timeout: Duration,
1929    /// Shared with the status handle so the attested value changes atomically
1930    /// when a rescan updates the running drain policy.
1931    effective_drain_timeout: Arc<Mutex<Duration>>,
1932    /// The supervisor-wide fallback, kept so a configuration update that
1933    /// REMOVES the per-module override can re-resolve to it.
1934    default_drain_timeout: Duration,
1935    health: HealthConfig,
1936    connection_file_path: Option<PathBuf>,
1937    capture_logs_dir: Option<PathBuf>,
1938    forwarding: Option<Arc<ForwardingTable>>,
1939    /// The shared handle, so every spawn path (initial, restart, reload) records the
1940    /// reserved-module launch nonce the HELLO verifier checks against.
1941    supervisor_handle: Option<SupervisorHandle>,
1942    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1943    /// status queries.
1944    ///
1945    /// One ring per module, held across every respawn. The lines explaining an exit
1946    /// are written BEFORE that exit, so a ring recreated per process would be empty
1947    /// exactly when it is asked for.
1948    stderr_ring: Arc<Mutex<StderrRing>>,
1949    terminal_ring: Arc<Mutex<TerminalRing>>,
1950    spawn_events: SpawnEventFeed,
1951    child_roster: ChildRoster,
1952    #[cfg(target_os = "linux")]
1953    cgroup_placement: Option<subc_cgroup::Placement>,
1954    #[cfg(test)]
1955    test_seed_stale_facts_before_enable_spawn: bool,
1956    #[cfg(test)]
1957    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962    spec: ModuleSpec,
1963    health: HealthConfig,
1964}
1965
1966/// Shared daemon lookup table for supervised module handles.
1967///
1968/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1969/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1970/// launch nonces recorded at spawn are checked by the same daemon instance.
1971#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976    /// Module ids the supervisor has taken on. An id is added BEFORE the
1977    /// module's first process is spawned and removed only when the module
1978    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1979    /// the keys of `modules`.
1980    ///
1981    /// `modules` cannot answer "is this module configured?" on its own: a
1982    /// [`SupervisedModule`] only exists once its process has been spawned, and
1983    /// a fast child can connect, register, sync its scopes and ask about them
1984    /// before the supervisor has inserted it. Answering "not configured" in that
1985    /// gap makes scope admission refuse with the terminal "will never sync"
1986    /// instead of the retryable "has not synced yet".
1987    configured_ids: Arc<Mutex<HashSet<String>>>,
1988    spawn_events: SpawnEventFeed,
1989    /// The current expected launch nonce for each reserved module_id. Set when the
1990    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1991    /// non-reserved module never has an entry here and is never nonce-checked.
1992    /// Reserved module ids and the nonce that authorizes their next HELLO.
1993    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1994    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1995    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1996    /// had NO entry and admitted anyone: the reservation protected the nonce
1997    /// holder, not the NAME (found live by CKCRED's canary probe registering
1998    /// against a reserved scratch id).
1999    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2001    ///
2002    /// This is deliberately in-memory only: subc is state-free across daemon
2003    /// restarts, and the tombstone only explains the hours-after-removal window
2004    /// while this executing daemon is still alive. Do not persist it in a store.
2005    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006    /// The current launch nonce for every supervised spawn. This is separate from
2007    /// reserved_nonces because consumer route.open attestation applies to all spawned
2008    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2009    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010    /// Reserved namespace prefixes mapped to the supervised owner module whose
2011    /// current spawn nonce authorizes HELLO claims below the prefix.
2012    ///
2013    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2014    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2015    /// accidental collisions and lower-trust processes from squatting protected
2016    /// namespaces.
2017    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018    /// Blue/green swaps in progress, by module id. An entry exists from just
2019    /// before the candidate process is spawned until the swap has failed, or
2020    /// has cut over and the old process is gone. While it exists, HELLO for the
2021    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2022    /// consumer attestation accepts both processes' nonces.
2023    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2025    promotion_observer: PromotionObserverSlot,
2026    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2027    /// this daemon-wide ordering, a rescan could retire or update a module while a
2028    /// concurrent reload still held its old handle and launch specification.
2029    operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032/// Told when a swap has promoted its candidate to be the module's active
2033/// registration.
2034///
2035/// An ordinary HELLO runs the control plane's registration side effects (the
2036/// capability cache, the deny census, the requirement recompute) as it
2037/// registers. A swap candidate's HELLO does not, because it is not routable;
2038/// promotion is when those must run instead, and promotion happens in the
2039/// supervisor, which has no other way into the control handler.
2040pub(crate) trait SwapPromotionObserver: Send + Sync {
2041    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2045/// control handler) owns this handle, so a strong reference back would be a
2046/// cycle that keeps both alive.
2047#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052        f.write_str("PromotionObserverSlot")
2053    }
2054}
2055
2056/// The nonces of one open swap.
2057#[derive(Debug, Clone)]
2058struct OpenSwap {
2059    /// The launch nonce minted for the candidate process. It is the swap
2060    /// token: the only thing that admits a HELLO into the candidate slot.
2061    candidate_nonce: String,
2062    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2063    /// here because cutover moves the module's recorded spawn nonce to the
2064    /// candidate while the incumbent is still draining and its consumers are
2065    /// still attesting with this one.
2066    incumbent_nonce: Option<String>,
2067    /// Set once a HELLO has been admitted with the swap token, so the token
2068    /// admits one registration and cannot be replayed after cutover empties
2069    /// the candidate slot.
2070    candidate_admitted: bool,
2071}
2072
2073/// What the swap gate says about a HELLO. See
2074/// [`SupervisorHandle::swap_hello_admission`].
2075#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077    /// No swap is open for the id (or the HELLO carries the incumbent's own
2078    /// nonce); the ordinary gates decide.
2079    NotSwapping,
2080    /// The HELLO carries the swap token: register it into the candidate slot.
2081    Candidate,
2082    /// A swap is open and the HELLO carries a nonce the supervisor did not
2083    /// mint for this id, no nonce, or a token already used.
2084    Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089    Exact {
2090        module_id: String,
2091    },
2092    Prefix {
2093        prefix: String,
2094        owner_module_id: String,
2095    },
2096}
2097
2098impl SupervisorHandle {
2099    pub fn new() -> Self {
2100        Self::default()
2101    }
2102
2103    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104        self.spawn_events.snapshot()
2105    }
2106
2107    pub(crate) fn subscribe_spawns(
2108        &self,
2109        connection_id: ConnectionId,
2110        corr: u64,
2111        version: u8,
2112        since: Option<SpawnCursor>,
2113        sink: FrameSink,
2114    ) -> Result<(), SpawnSubscribeRefusal> {
2115        self.spawn_events
2116            .subscribe(connection_id, corr, version, since, sink)
2117    }
2118
2119    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120        self.spawn_events.cancel(connection_id, corr)
2121    }
2122
2123    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124        self.spawn_events.remove_connection(connection_id);
2125    }
2126
2127    #[cfg(any(test, feature = "test-support"))]
2128    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129        assert!(capacity > 0, "spawn event capacity must be non-zero");
2130        self.spawn_events.set_capacity(capacity);
2131    }
2132
2133    #[cfg(any(test, feature = "test-support"))]
2134    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135        self.spawn_events.subscriber_count()
2136    }
2137
2138    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2139    /// a respawn invalidates stale consumer identities.
2140    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141        self.spawn_nonces
2142            .lock()
2143            .unwrap_or_else(|poisoned| poisoned.into_inner())
2144            .insert(module_id.to_string(), nonce);
2145    }
2146
2147    /// Record the launch nonce expected from the next HELLO for a reserved module,
2148    /// replacing any prior nonce (a respawn invalidates the previous one).
2149    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150        self.reserved_nonces
2151            .lock()
2152            .unwrap_or_else(|poisoned| poisoned.into_inner())
2153            .insert(module_id.to_string(), Some(nonce));
2154    }
2155
2156    /// Record namespace prefixes owned by a supervised module.
2157    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158        let mut owners = self
2159            .reserved_prefix_owners
2160            .lock()
2161            .unwrap_or_else(|poisoned| poisoned.into_inner());
2162        owners.retain(|_, owner| owner != owner_module_id);
2163        for prefix in prefixes {
2164            owners.insert(prefix.clone(), owner_module_id.to_string());
2165        }
2166    }
2167
2168    /// The launch nonce most recently minted for a module's spawn, if any.
2169    #[cfg(test)]
2170    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171        self.spawn_nonces
2172            .lock()
2173            .unwrap_or_else(|poisoned| poisoned.into_inner())
2174            .get(module_id)
2175            .cloned()
2176    }
2177
2178    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180        let spawn_nonce = self
2181            .spawn_nonces
2182            .lock()
2183            .unwrap_or_else(|poisoned| poisoned.into_inner())
2184            .get(&spec.module_id)
2185            .cloned();
2186        let mut reserved_nonces = self
2187            .reserved_nonces
2188            .lock()
2189            .unwrap_or_else(|poisoned| poisoned.into_inner());
2190        if spec.reserved {
2191            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2192            // reserved name whose module has never spawned has no legitimate
2193            // holder, and the entry's absence is what used to leave the name
2194            // open to the first claimant.
2195            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196        }
2197        drop(reserved_nonces);
2198        // A later unreserved declaration must not silently unreserve an id that
2199        // was retained after its reserved configuration was removed. The explicit
2200        // release ceremony is the only operation that retires that gate.
2201        self.removal_tombstones
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .remove(&spec.module_id);
2205    }
2206
2207    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2208    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2209    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2210    /// with no matching prefix are always authorized.
2211    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212        self.reserved_hello_verification(module_id, presented)
2213            .is_ok()
2214    }
2215
2216    /// Whether HELLO passed a nonce check that was actually required by reserved
2217    /// module policy, or the refusal explaining why a required check failed.
2218    pub(crate) fn reserved_hello_verification(
2219        &self,
2220        module_id: &str,
2221        presented: Option<&str>,
2222    ) -> Result<bool, ReservedHelloRejection> {
2223        let nonces = self
2224            .reserved_nonces
2225            .lock()
2226            .unwrap_or_else(|poisoned| poisoned.into_inner());
2227        if let Some(expected) = nonces.get(module_id) {
2228            // `None` = reserved with no legitimate holder: refuse every
2229            // presentation, because no process can hold a nonce that was never
2230            // minted. Only a real minted nonce admits, in constant time.
2231            let authorized = match expected {
2232                Some(expected) => {
2233                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2234                }
2235                None => false,
2236            };
2237            if authorized {
2238                return Ok(true);
2239            }
2240            return Err(ReservedHelloRejection::Exact {
2241                module_id: module_id.to_string(),
2242            });
2243        }
2244        drop(nonces);
2245
2246        let matched_prefix = self
2247            .reserved_prefix_owners
2248            .lock()
2249            .unwrap_or_else(|poisoned| poisoned.into_inner())
2250            .iter()
2251            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2252            .max_by_key(|(prefix, _)| prefix.len())
2253            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2254        let Some((prefix, owner_module_id)) = matched_prefix else {
2255            return Ok(false);
2256        };
2257
2258        let authorized = presented.is_some_and(|presented| {
2259            self.spawn_nonces
2260                .lock()
2261                .unwrap_or_else(|poisoned| poisoned.into_inner())
2262                .get(&owner_module_id)
2263                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2264                // While the owner is being swapped, children started by
2265                // either of its two processes hold that process's nonce.
2266                || self.swap_nonce_matches(&owner_module_id, presented)
2267        });
2268        if authorized {
2269            Ok(true)
2270        } else {
2271            Err(ReservedHelloRejection::Prefix {
2272                prefix,
2273                owner_module_id,
2274            })
2275        }
2276    }
2277
2278    /// Whether a consumer connection proved it came from a daemon-spawned module.
2279    ///
2280    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2281    /// accepted only for module ids the supervisor has spawned.
2282    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2283        if presented.is_empty() {
2284            return false;
2285        }
2286        let nonces = self
2287            .spawn_nonces
2288            .lock()
2289            .unwrap_or_else(|poisoned| poisoned.into_inner());
2290        let current = nonces
2291            .get(module_id)
2292            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2293        drop(nonces);
2294        // During a swap two processes of the module are alive, and a consumer
2295        // started by either one presents that process's nonce. Accepting only
2296        // the recorded one would fail the incumbent's consumers for the whole
2297        // overlap once cutover moves the record to the candidate.
2298        current || self.swap_nonce_matches(module_id, presented)
2299    }
2300
2301    /// Whether `presented` is either nonce of an open swap for `module_id`.
2302    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2303        let swaps = self
2304            .swaps
2305            .lock()
2306            .unwrap_or_else(|poisoned| poisoned.into_inner());
2307        swaps.get(module_id).is_some_and(|swap| {
2308            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2309                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2310                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2311                })
2312        })
2313    }
2314
2315    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2316    /// Called before the candidate process exists.
2317    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2318        let incumbent_nonce = self
2319            .spawn_nonces
2320            .lock()
2321            .unwrap_or_else(|poisoned| poisoned.into_inner())
2322            .get(module_id)
2323            .cloned();
2324        self.swaps
2325            .lock()
2326            .unwrap_or_else(|poisoned| poisoned.into_inner())
2327            .insert(
2328                module_id.to_string(),
2329                OpenSwap {
2330                    candidate_nonce,
2331                    incumbent_nonce,
2332                    candidate_admitted: false,
2333                },
2334            );
2335    }
2336
2337    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2338    /// the module's recorded one.
2339    pub(crate) fn close_swap(&self, module_id: &str) {
2340        self.swaps
2341            .lock()
2342            .unwrap_or_else(|poisoned| poisoned.into_inner())
2343            .remove(module_id);
2344    }
2345
2346    /// Install the observer told about swap promotions, replacing any earlier
2347    /// one.
2348    pub(crate) fn set_swap_promotion_observer(
2349        &self,
2350        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2351    ) {
2352        *self
2353            .promotion_observer
2354            .0
2355            .lock()
2356            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2357    }
2358
2359    /// Tell the installed observer, if it is still alive, that a swap promoted
2360    /// `registration`.
2361    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2362        let observer = self
2363            .promotion_observer
2364            .0
2365            .lock()
2366            .unwrap_or_else(|poisoned| poisoned.into_inner())
2367            .as_ref()
2368            .and_then(std::sync::Weak::upgrade);
2369        if let Some(observer) = observer {
2370            observer.swap_promoted(registration);
2371        }
2372    }
2373
2374    /// Whether a swap is open for `module_id`.
2375    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2376        self.swaps
2377            .lock()
2378            .unwrap_or_else(|poisoned| poisoned.into_inner())
2379            .contains_key(module_id)
2380    }
2381
2382    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2383    /// respawn would, once cutover has made the candidate the module's process.
2384    /// The swap stays open so the incumbent's nonce keeps attesting until the
2385    /// incumbent has drained and exited.
2386    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2387        let candidate_nonce = self
2388            .swaps
2389            .lock()
2390            .unwrap_or_else(|poisoned| poisoned.into_inner())
2391            .get(module_id)
2392            .map(|swap| swap.candidate_nonce.clone());
2393        let Some(nonce) = candidate_nonce else {
2394            return;
2395        };
2396        self.set_spawn_nonce(module_id, nonce.clone());
2397        if reserved {
2398            self.set_reserved_nonce(module_id, nonce);
2399        }
2400    }
2401
2402    /// The swap gate for a HELLO claiming `module_id`.
2403    ///
2404    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2405    /// presents the candidate nonce, which the reserved gate (holding the
2406    /// incumbent's nonce) would refuse as `reserved_module` before swap
2407    /// admission was ever reached. And it applies to unreserved ids too: for an
2408    /// unreserved id the only thing that ever stopped a second process claiming
2409    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2410    /// refusal a swap lifts for its candidate.
2411    ///
2412    /// The incumbent's own nonce falls through to the ordinary gates, which
2413    /// treat it as they always have (a live incumbent is refused as a
2414    /// duplicate). Anything else while a swap is open is refused, including an
2415    /// absent nonce.
2416    pub(crate) fn swap_hello_admission(
2417        &self,
2418        module_id: &str,
2419        presented: Option<&str>,
2420    ) -> SwapHelloAdmission {
2421        let swaps = self
2422            .swaps
2423            .lock()
2424            .unwrap_or_else(|poisoned| poisoned.into_inner());
2425        let Some(swap) = swaps.get(module_id) else {
2426            return SwapHelloAdmission::NotSwapping;
2427        };
2428        let Some(presented) = presented else {
2429            return SwapHelloAdmission::Refused;
2430        };
2431        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2432            return if swap.candidate_admitted {
2433                SwapHelloAdmission::Refused
2434            } else {
2435                SwapHelloAdmission::Candidate
2436            };
2437        }
2438        if swap
2439            .incumbent_nonce
2440            .as_deref()
2441            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2442        {
2443            return SwapHelloAdmission::NotSwapping;
2444        }
2445        SwapHelloAdmission::Refused
2446    }
2447
2448    /// Record that the swap token has registered a candidate, so it admits no
2449    /// second HELLO.
2450    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2451        if let Some(swap) = self
2452            .swaps
2453            .lock()
2454            .unwrap_or_else(|poisoned| poisoned.into_inner())
2455            .get_mut(module_id)
2456        {
2457            swap.candidate_admitted = true;
2458        }
2459    }
2460
2461    /// Test/support lookup for the current launch nonce of a supervised spawn.
2462    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2463        self.spawn_nonces
2464            .lock()
2465            .unwrap_or_else(|poisoned| poisoned.into_inner())
2466            .get(module_id)
2467            .cloned()
2468    }
2469
2470    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2471    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2472        self.reserved_nonces
2473            .lock()
2474            .unwrap_or_else(|poisoned| poisoned.into_inner())
2475            .get(module_id)
2476            .cloned()
2477            .flatten()
2478    }
2479
2480    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2481        // Normally already marked before the process was spawned; marking here
2482        // too keeps `configured_ids` a superset of the roster for any caller
2483        // that inserts a module directly.
2484        self.mark_configured(module.module_id());
2485        let mut modules = self
2486            .modules
2487            .lock()
2488            .unwrap_or_else(|poisoned| poisoned.into_inner());
2489        modules.insert(module.module_id().to_string(), module)
2490    }
2491
2492    /// Record that the supervisor has taken on `module_id`. Called before the
2493    /// module's first process is spawned, so that by the time that process can
2494    /// register, [`Self::is_configured`] already answers true.
2495    fn mark_configured(&self, module_id: &str) {
2496        self.configured_ids
2497            .lock()
2498            .unwrap_or_else(|poisoned| poisoned.into_inner())
2499            .insert(module_id.to_string());
2500    }
2501
2502    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2503    /// before it was ever put on the roster. A module already on the roster
2504    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2505    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2506        let modules = self
2507            .modules
2508            .lock()
2509            .unwrap_or_else(|poisoned| poisoned.into_inner());
2510        if !modules.contains_key(module_id) {
2511            self.configured_ids
2512                .lock()
2513                .unwrap_or_else(|poisoned| poisoned.into_inner())
2514                .remove(module_id);
2515        }
2516    }
2517
2518    /// Whether `module_id` is a module this daemon supervises: on the roster,
2519    /// or about to be (its process is being spawned right now).
2520    ///
2521    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2522    /// for scopes: a supervised module's process can register and sync before
2523    /// [`Self::get`] can return it, and in that window it is still a module
2524    /// that will sync, not one that never will.
2525    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2526        self.configured_ids
2527            .lock()
2528            .unwrap_or_else(|poisoned| poisoned.into_inner())
2529            .contains(module_id)
2530    }
2531
2532    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2533        let modules = self
2534            .modules
2535            .lock()
2536            .unwrap_or_else(|poisoned| poisoned.into_inner());
2537        modules.get(module_id).cloned()
2538    }
2539
2540    pub(crate) fn record_late_health_answer(
2541        &self,
2542        module_id: &str,
2543        latency_ms: u64,
2544    ) -> Result<bool, SuperviseError> {
2545        let Some(module) = self.get(module_id) else {
2546            return Ok(false);
2547        };
2548        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2549            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2550            state.health.last_late_answer_latency_ms = Some(latency_ms);
2551            // A late answer is an answer: the module served the probe, just past
2552            // the deadline. Leaving the miss streak in place while logging
2553            // "proves the module is alive" is how a CPU-starved module that
2554            // answers every probe a few seconds late still marches to the
2555            // threshold and gets killed — the exact kill class `NoAnswer` is
2556            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2557            // is degradation, and degradation reports; it does not restart.
2558            state.health.consecutive_failures = 0;
2559        })?;
2560        Ok(true)
2561    }
2562
2563    /// Arm the one-shot marker for the module process that this caller
2564    /// deliberately initiated severance against. Generic connection teardown
2565    /// must not call this:
2566    /// a surviving process would otherwise retain an exemption for a later
2567    /// genuine crash.
2568    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2569        let Some(module) = self.get(module_id) else {
2570            return Ok(false);
2571        };
2572        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2573        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2574            return Ok(false);
2575        };
2576        drop(snapshot);
2577        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2578    }
2579
2580    pub fn list(&self) -> Vec<SupervisedModule> {
2581        let modules = self
2582            .modules
2583            .lock()
2584            .unwrap_or_else(|poisoned| poisoned.into_inner());
2585        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2586        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2587        modules
2588    }
2589
2590    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2591        self.activity_images
2592            .lock()
2593            .unwrap_or_else(|p| p.into_inner())
2594            .remove(module_id);
2595        self.spawn_nonces
2596            .lock()
2597            .unwrap_or_else(|poisoned| poisoned.into_inner())
2598            .remove(module_id);
2599        self.close_swap(module_id);
2600        let mut reserved_nonces = self
2601            .reserved_nonces
2602            .lock()
2603            .unwrap_or_else(|poisoned| poisoned.into_inner());
2604        if reserved_nonces.contains_key(module_id) {
2605            // The old nonce must die with the removed process, but the exact-id
2606            // gate remains until an operator explicitly releases it.
2607            reserved_nonces.insert(module_id.to_string(), None);
2608        }
2609        drop(reserved_nonces);
2610        self.reserved_prefix_owners
2611            .lock()
2612            .unwrap_or_else(|poisoned| poisoned.into_inner())
2613            .retain(|_, owner| owner != module_id);
2614        let removed = self
2615            .modules
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        self.configured_ids
2620            .lock()
2621            .unwrap_or_else(|poisoned| poisoned.into_inner())
2622            .remove(module_id);
2623        removed
2624    }
2625
2626    /// Remember a module removed by a non-preview rescan so route.open can
2627    /// distinguish that intentional removal from an unknown id.
2628    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2629        self.removal_tombstones
2630            .lock()
2631            .unwrap_or_else(|poisoned| poisoned.into_inner())
2632            .insert(module_id.to_string(), unix_ms_now());
2633    }
2634
2635    /// Return how long ago a rescan removed this module in milliseconds.
2636    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2637        self.removal_tombstones
2638            .lock()
2639            .unwrap_or_else(|poisoned| poisoned.into_inner())
2640            .get(module_id)
2641            .copied()
2642            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2643    }
2644
2645    /// Retire a reserved-id gate only after its module has left supervision.
2646    ///
2647    /// A retained gate has no live nonce (`None`), so releasing any other entry
2648    /// would weaken a currently configured or otherwise active reservation.
2649    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2650        if self.get(module_id).is_some() {
2651            return false;
2652        }
2653        let mut reserved_nonces = self
2654            .reserved_nonces
2655            .lock()
2656            .unwrap_or_else(|poisoned| poisoned.into_inner());
2657        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2658            return false;
2659        }
2660        reserved_nonces.remove(module_id);
2661        true
2662    }
2663
2664    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2665        Arc::clone(&self.operation_lock)
2666    }
2667}
2668
2669/// Process supervisor for subc-owned singleton modules.
2670#[derive(Debug, Clone)]
2671pub struct Supervisor {
2672    registry: Arc<Registry>,
2673    restart_policy: RestartPolicy,
2674    drain_timeout: Duration,
2675    connection_file_path: Option<PathBuf>,
2676    capture_logs_dir: Option<PathBuf>,
2677    forwarding: Option<Arc<ForwardingTable>>,
2678    process_liveness: Arc<SupervisorProcessLiveness>,
2679    supervisor_handle: Option<SupervisorHandle>,
2680    health: HealthConfig,
2681    daemon_start_clock: crate::clock::StartClock,
2682    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2683    spawn_events: SpawnEventFeed,
2684    provenance_probe: ExecutableIdentityProbe,
2685    /// Every process spawned through this supervisor (and its clones) and not
2686    /// yet reaped, so daemon shutdown can end them.
2687    child_roster: ChildRoster,
2688    #[cfg(target_os = "linux")]
2689    cgroup_placement: Option<subc_cgroup::Placement>,
2690    #[cfg(test)]
2691    test_after_first_spawn: AfterFirstSpawnHook,
2692}
2693
2694/// A roster entry whose monitor and first process have not started yet.
2695/// Keeping the command receiver here lets discovery serve status immediately
2696/// while start/stop requests wait until bootstrap has published the connection file.
2697pub(crate) struct RegisteredConfiguredModule {
2698    spec: ModuleSpec,
2699    runtime: SupervisorRuntimeConfig,
2700    module: SupervisedModule,
2701    commands: mpsc::Receiver<SupervisorCommand>,
2702}
2703
2704/// Test-only hook run on the path that takes on a new module, right after its
2705/// first `spawn_child` returns (the process exists and could already be
2706/// registering) and before that process is handed to the module's supervise
2707/// loop and put on the roster. Lets a test observe what a fast child would see
2708/// in that window without racing a real one.
2709#[cfg(test)]
2710#[derive(Clone, Default)]
2711struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2712
2713#[cfg(test)]
2714type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2715
2716#[cfg(test)]
2717impl fmt::Debug for AfterFirstSpawnHook {
2718    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2719        f.write_str("AfterFirstSpawnHook")
2720    }
2721}
2722
2723#[cfg(test)]
2724impl AfterFirstSpawnHook {
2725    fn run(&self, module_id: &str) {
2726        if let Some(hook) = &self.0 {
2727            hook(module_id);
2728        }
2729    }
2730}
2731
2732impl Supervisor {
2733    #[cfg(test)]
2734    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2735        let supervisor = Self::new(registry, policy);
2736        #[cfg(target_os = "macos")]
2737        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2738        supervisor
2739    }
2740    /// Verify the trampoline once when configured. Missing private OS support
2741    /// refuses every macOS launch by name but does not stop the daemon's control
2742    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2743    /// entry point; the library must not exec an arbitrary hosting program.
2744    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2745        let path = path.into();
2746        #[cfg(target_os = "macos")]
2747        {
2748            let result = probe_privacy_trampoline(&path).map(|()| path);
2749            if let Err(cause) = &result {
2750                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2751            }
2752            self.child_roster.set_privacy_trampoline(result);
2753        }
2754        #[cfg(not(target_os = "macos"))]
2755        let _ = path;
2756        self
2757    }
2758    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2759    /// capability-requirement log to read.
2760    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2761        self.child_roster.shutdown_flag()
2762    }
2763
2764    /// The first step of an announced daemon shutdown, before the notice and
2765    /// before any connection is closed.
2766    ///
2767    /// Sets the daemon-shutdown flag first: from here on no module is
2768    /// respawned (crash restart, operator restart, or swap), and every child
2769    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2770    /// the module exits on the EOF this shutdown gives it or is signalled by a
2771    /// service manager that kills the whole cgroup. Then writes the journal's
2772    /// shutdown marker, which records the instant and closes this daemon
2773    /// incarnation's stretch of the journal.
2774    #[cfg(unix)]
2775    pub(crate) fn begin_daemon_shutdown(&self) {
2776        self.child_roster.close();
2777        if let Some(journal) = &self.terminal_journal {
2778            journal.stamp_shutdown();
2779        }
2780    }
2781
2782    /// Announce a cut while established connections can still carry replies.
2783    /// These budgets promise notice and a bounded wait, not child completion;
2784    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2785    #[cfg(unix)]
2786    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2787        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2788        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2789        let Some(forwarding) = &self.forwarding else {
2790            return Ok(());
2791        };
2792        let notice_started = Instant::now();
2793        let modules = forwarding
2794            .begin_daemon_drain()
2795            .map_err(SuperviseError::Forwarding)?;
2796        let deadline_ms =
2797            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2798        let mut notices = tokio::task::JoinSet::new();
2799        let mut drains = Vec::new();
2800        for (module_id, endpoint) in modules {
2801            let Some(target) = forwarding
2802                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2803                .map_err(SuperviseError::Forwarding)?
2804            else {
2805                continue;
2806            };
2807            let routes = forwarding
2808                .endpoint_routes(target.endpoint)
2809                .map_err(SuperviseError::Forwarding)?;
2810            // Restart allows deployed consumers to reopen after the new daemon
2811            // appears. The wire reason stays `restart`; what tells a daemon cut
2812            // apart from a module restart afterwards is the terminal record
2813            // itself, whose disposition is `daemon_shutdown` for every exit
2814            // observed once `begin_daemon_shutdown` has run.
2815            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2816                reason: RouteCloseReason::Restart,
2817                deadline_ms,
2818            })
2819            .expect("module draining serializes");
2820            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2821            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2822            for route in routes {
2823                let client = route.goodbye_target;
2824                if let Some((_, channels)) = clients
2825                    .iter_mut()
2826                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2827                {
2828                    channels.push(client.channel);
2829                } else {
2830                    let channel = client.channel;
2831                    clients.push((client, vec![channel]));
2832                }
2833            }
2834            for (client, mut channels) in clients {
2835                channels.sort_unstable();
2836                channels.dedup();
2837                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2838                    module_id: module_id.clone(),
2839                    channels,
2840                    reason: RouteCloseReason::Restart,
2841                })
2842                .expect("route closing serializes");
2843                recipients.push((client.sink, client.negotiated_ver, closing));
2844            }
2845            for (sink, version, body) in recipients {
2846                notices.spawn(async move {
2847                    let frame = Frame::build_with_version(
2848                        version,
2849                        FrameType::Push,
2850                        control_flags(),
2851                        0,
2852                        0,
2853                        0,
2854                        body,
2855                    )
2856                    .expect("bounded lifecycle notice frame builds");
2857                    sink.send_flushed(frame).await
2858                });
2859            }
2860            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2861            drains.push((module_id, target.endpoint, gauges));
2862        }
2863        // A quiet forwarding table is not proof that queued notices reached the
2864        // socket. Wait for writer flush acknowledgements before testing quiescence.
2865        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2866        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2867            if !matches!(result, Ok(Ok(()))) {
2868                warn!(?result, "daemon shutdown notice delivery failed");
2869            }
2870        }
2871        notices.abort_all();
2872        let notice_elapsed = notice_started.elapsed();
2873        let drain_started = Instant::now();
2874        let deadline = drain_started + DRAIN_BUDGET;
2875        let mut waits = tokio::task::JoinSet::new();
2876        for (module_id, endpoint, gauges) in drains {
2877            let forwarding = Arc::clone(forwarding);
2878            let mut runtime = self.runtime_config();
2879            runtime.health.cadence = Duration::from_millis(100);
2880            waits.spawn(async move {
2881                let wait_result = wait_for_forwarding_quiescence(
2882                    &forwarding,
2883                    &module_id,
2884                    &runtime,
2885                    endpoint,
2886                    deadline,
2887                    &gauges,
2888                    DrainScope::Endpoint(endpoint),
2889                    &mut DrainGaugeTrace::new(&gauges),
2890                )
2891                .await;
2892                // Store the measurement before the task returns: quiescence
2893                // may be reached just as the deadline stops the join loop from
2894                // collecting results. Unfinished or failed waits report false.
2895                forwarding
2896                    .record_daemon_drain_result(
2897                        endpoint,
2898                        drained_after_quiescence_wait(&wait_result),
2899                    )
2900                    .map_err(SuperviseError::Forwarding)?;
2901                wait_result
2902            });
2903        }
2904        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2905            if !matches!(result, Ok(Ok(true))) {
2906                warn!(?result, "daemon shutdown drain did not reach quiescence");
2907            }
2908        }
2909        // Measure only the forwarding drain, not child teardown or the caller's
2910        // signal delivery and process-reaping delays.
2911        info!(
2912            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2913            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2914            "daemon shutdown forwarding drain complete"
2915        );
2916        Ok(())
2917    }
2918
2919    /// The last step of an announced daemon shutdown, after the notice and the
2920    /// drain: send every registered module a module GOODBYE, the same planned
2921    /// stop signal `ck module stop` gives, then close every connection so each
2922    /// subc module sees EOF and starts its own teardown, then end every
2923    /// supervised child that has not exited
2924    /// by its own deadline (its drain budget, capped). Modules lead their own
2925    /// process groups, so a
2926    /// service manager's group kill no longer reaches them; without this a
2927    /// child that does not stop on EOF (every `protocol: "none"` child, which
2928    /// has no connection) would outlive the daemon. Every wait is bounded (see
2929    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2930    #[cfg(unix)]
2931    pub(crate) async fn end_children_for_daemon_shutdown(
2932        &self,
2933        already_escalated: bool,
2934        escalate: impl std::future::Future<Output = ()>,
2935    ) {
2936        tokio::pin!(escalate);
2937        let mut escalated = already_escalated;
2938        if let Some(forwarding) = &self.forwarding {
2939            let reason = CloseReason::new(
2940                "daemon_shutdown",
2941                "the daemon is exiting after its shutdown notice and drain",
2942            );
2943            if escalated {
2944                // The operator asked to stop waiting: queue the GOODBYEs but
2945                // do not wait for them to be written.
2946                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2947            } else {
2948                tokio::select! {
2949                    biased;
2950                    _ = escalate.as_mut() => {
2951                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2952                        escalated = true;
2953                    }
2954                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2955                }
2956            }
2957            let closed = forwarding.close_all_connections(&reason);
2958            debug!(closed, "closed established connections for daemon shutdown");
2959        }
2960        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2961        // already completed and must not be polled again; the child shutdown
2962        // wait is told it is escalated and gets a future that never fires.
2963        let escalated_here = escalated && !already_escalated;
2964        let remaining_escalate = async move {
2965            if escalated_here {
2966                std::future::pending::<()>().await;
2967            } else {
2968                escalate.await;
2969            }
2970        };
2971        crate::child_roster::end_children_for_daemon_shutdown(
2972            &self.child_roster,
2973            escalated,
2974            remaining_escalate,
2975        )
2976        .await;
2977    }
2978
2979    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2980        Self {
2981            registry,
2982            restart_policy,
2983            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2984            connection_file_path: None,
2985            capture_logs_dir: None,
2986            forwarding: None,
2987            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2988            supervisor_handle: None,
2989            health: HealthConfig::default(),
2990            daemon_start_clock: crate::clock::StartClock::capture(),
2991            terminal_journal: None,
2992            spawn_events: SpawnEventFeed::default(),
2993            provenance_probe: ExecutableIdentityProbe::default(),
2994            child_roster: ChildRoster::default(),
2995            #[cfg(target_os = "linux")]
2996            cgroup_placement: None,
2997            #[cfg(test)]
2998            test_after_first_spawn: AfterFirstSpawnHook::default(),
2999        }
3000    }
3001
3002    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3003        self.drain_timeout = drain_timeout;
3004        self
3005    }
3006
3007    /// The drain budget a module gets when its config entry sets none.
3008    pub(crate) fn default_drain_timeout(&self) -> Duration {
3009        self.drain_timeout
3010    }
3011
3012    pub fn with_process_liveness(
3013        mut self,
3014        process_liveness: Arc<SupervisorProcessLiveness>,
3015    ) -> Self {
3016        self.process_liveness = process_liveness;
3017        self
3018    }
3019
3020    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3021        self.connection_file_path = Some(connection_file_path.into());
3022        self
3023    }
3024
3025    /// Enables daemon-owned capture files for supervised stdout and stderr.
3026    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3027        self.capture_logs_dir = Some(logs_dir.into());
3028        self
3029    }
3030
3031    /// Names this daemon lifetime in spawn events, independently of whether a
3032    /// terminal journal is configured.
3033    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3034        // A millisecond start stamp can repeat after clock rollback or a rapid
3035        // restart. Use the connection file's random daemon_id instead: it already
3036        // identifies this daemon lifetime independently of the wall clock.
3037        self.spawn_events.configure_incarnation(daemon_incarnation);
3038        self
3039    }
3040
3041    /// Enables best-effort history shared by every supervised module. Without
3042    /// it, terminal history is kept only in each module's in-memory ring.
3043    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3044        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3045        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3046            path,
3047            daemon_incarnation,
3048        )));
3049        this
3050    }
3051
3052    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3053        self.forwarding = Some(forwarding);
3054        self
3055    }
3056
3057    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3058        self.spawn_events = supervisor_handle.spawn_events.clone();
3059        self.supervisor_handle = Some(supervisor_handle);
3060        self
3061    }
3062
3063    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3064        self.health = health;
3065        self
3066    }
3067
3068    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3069    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3070    /// record is kept.
3071    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3072        self.child_roster.record_to(path.into());
3073        self
3074    }
3075
3076    #[cfg(target_os = "linux")]
3077    pub fn with_cgroup_placement(
3078        mut self,
3079        cgroup_placement: Option<subc_cgroup::Placement>,
3080    ) -> Self {
3081        self.cgroup_placement = cgroup_placement;
3082        self
3083    }
3084
3085    /// Spawn `spec.program` and start monitoring it.
3086    ///
3087    /// The child is expected to parse `--subc <connection-file-path>`, read the
3088    /// TCP+key connection file, authenticate to the already-running listener, and
3089    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3090    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3091        validate_spec(&spec)?;
3092        self.establish_identity(&spec);
3093
3094        let runtime = self.runtime_config();
3095        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3096        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3097        let spawned = spawn_child(
3098            &spec,
3099            runtime.connection_file_path.as_deref(),
3100            self.supervisor_handle.as_ref(),
3101            &runtime.stderr_ring,
3102            runtime.capture_logs_dir.as_deref(),
3103            &runtime.child_roster,
3104            #[cfg(target_os = "linux")]
3105            runtime.cgroup_placement.as_ref(),
3106        );
3107        #[cfg(test)]
3108        self.test_after_first_spawn.run(&spec.module_id);
3109        let child = match spawned {
3110            Ok(child) => child,
3111            Err(err) => {
3112                // Unlike the configured paths, a failed `spawn` leaves nothing
3113                // on the roster, so the module must not stay marked configured.
3114                self.abandon_unrostered(&spec.module_id);
3115                return Err(err);
3116            }
3117        };
3118        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3119        if let Some(spawn) = activity_spawn {
3120            spawn.complete(&snapshot);
3121        }
3122
3123        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3124    }
3125
3126    /// Make `spec`'s module count as configured, with its identity gates
3127    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3128    /// exists.
3129    ///
3130    /// Every path that takes on a new module calls this before `spawn_child`.
3131    /// The order is the point: the child can connect, register, sync its
3132    /// scopes and ask about them as soon as it is spawned, and the module is
3133    /// only put on the roster after `spawn_child` returns. Were the mark set
3134    /// with the roster entry, a fast child would see its own owner reported
3135    /// as not configured, and a scoped `route.open` in that window would be
3136    /// refused as terminal `scope_not_live` ("will never sync") instead of
3137    /// retryable `scope_not_synced`.
3138    fn establish_identity(&self, spec: &ModuleSpec) {
3139        if let Some(supervisor_handle) = &self.supervisor_handle {
3140            supervisor_handle.apply_identity_configuration(spec);
3141            supervisor_handle.mark_configured(&spec.module_id);
3142        }
3143    }
3144
3145    /// Take back [`Self::establish_identity`]'s configured mark when the
3146    /// module will not be put on the roster after all.
3147    fn abandon_unrostered(&self, module_id: &str) {
3148        if let Some(supervisor_handle) = &self.supervisor_handle {
3149            supervisor_handle.unmark_configured_unless_rostered(module_id);
3150        }
3151    }
3152
3153    /// Record a freshly spawned first process as running. On failure the
3154    /// module never reaches the roster, so its configured mark is taken back.
3155    fn mark_first_process_running(
3156        &self,
3157        spec: &ModuleSpec,
3158        runtime: &SupervisorRuntimeConfig,
3159        snapshot: &SharedSnapshot,
3160        child: &SupervisedChild,
3161    ) -> Result<(), SuperviseError> {
3162        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3163            self.abandon_unrostered(&spec.module_id);
3164            return Err(err);
3165        }
3166        self.process_liveness
3167            .track(spec.module_id.clone(), Arc::clone(snapshot));
3168        Ok(())
3169    }
3170
3171    /// Start supervising a module declared in daemon configuration.
3172    ///
3173    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3174    /// failures in the supervisor handle so operator-facing `supervisor.list`
3175    /// reflects every configured module while daemon startup continues.
3176    pub fn supervise_configured(
3177        &self,
3178        spec: ModuleSpec,
3179        enabled: bool,
3180    ) -> Result<SupervisedModule, SuperviseError> {
3181        self.supervise_configured_requested(
3182            spec,
3183            enabled,
3184            self.health.clone(),
3185            None,
3186            self.restart_policy,
3187            None,
3188        )
3189    }
3190
3191    /// Supervise a configured module with its own health, drain, and crash
3192    /// budget. The restart policy is per-module because the config file is:
3193    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3194    /// module that is expensive to restart should not be forced onto the same
3195    /// budget as one that is cheap.
3196    pub fn supervise_configured_with_health(
3197        &self,
3198        spec: ModuleSpec,
3199        enabled: bool,
3200        health: HealthConfig,
3201        drain_timeout_ms: Option<u64>,
3202        restart_policy: RestartPolicy,
3203    ) -> Result<SupervisedModule, SuperviseError> {
3204        self.supervise_configured_requested(
3205            spec,
3206            enabled,
3207            health,
3208            drain_timeout_ms,
3209            restart_policy,
3210            None,
3211        )
3212    }
3213
3214    pub(crate) fn supervise_configured_requested(
3215        &self,
3216        spec: ModuleSpec,
3217        enabled: bool,
3218        health: HealthConfig,
3219        drain_timeout_ms: Option<u64>,
3220        restart_policy: RestartPolicy,
3221        requester: Option<Principal>,
3222    ) -> Result<SupervisedModule, SuperviseError> {
3223        let registered = self.register_configured_with_health(
3224            spec,
3225            enabled,
3226            health,
3227            drain_timeout_ms,
3228            restart_policy,
3229        )?;
3230        self.start_registered_configured(registered, requester)
3231    }
3232
3233    /// Record a configured module as starting or disabled, without launching a
3234    /// process or running its monitor. Bootstrap registers the entire roster
3235    /// before publishing discovery, then activates it once children can read
3236    /// the connection file passed in their `--subc` argument.
3237    pub(crate) fn register_configured_with_health(
3238        &self,
3239        spec: ModuleSpec,
3240        enabled: bool,
3241        health: HealthConfig,
3242        drain_timeout_ms: Option<u64>,
3243        restart_policy: RestartPolicy,
3244    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3245        validate_spec(&spec)?;
3246        self.establish_identity(&spec);
3247
3248        let mut runtime = self.runtime_config();
3249        runtime.health = health;
3250        runtime.restart_policy = restart_policy;
3251        if let Some(ms) = drain_timeout_ms {
3252            runtime.drain_timeout = Duration::from_millis(ms);
3253            *runtime
3254                .effective_drain_timeout
3255                .lock()
3256                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3257        }
3258        let snapshot = if enabled {
3259            SupervisorSnapshot::starting()
3260        } else {
3261            SupervisorSnapshot::disabled()
3262        };
3263        let snapshot = {
3264            let mut snapshot = snapshot;
3265            if enabled {
3266                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3267                {
3268                    snapshot.waiting_for_previous_pid = Some(pid);
3269                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3270                }
3271            }
3272            snapshot
3273        };
3274        let (module, commands) =
3275            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3276        Ok(RegisteredConfiguredModule {
3277            spec,
3278            runtime,
3279            module,
3280            commands,
3281        })
3282    }
3283
3284    /// Activate a previously registered module. Queued control requests are
3285    /// served by the monitor after the initial configured spawn has settled.
3286    pub(crate) fn start_registered_configured(
3287        &self,
3288        registered: RegisteredConfiguredModule,
3289        requester: Option<Principal>,
3290    ) -> Result<SupervisedModule, SuperviseError> {
3291        let RegisteredConfiguredModule {
3292            spec,
3293            mut runtime,
3294            module,
3295            commands,
3296        } = registered;
3297        runtime.start_requester = requester;
3298        let snapshot = &module.inner.snapshot;
3299        let should_spawn = {
3300            let state = lock_snapshot(snapshot)?;
3301            state.enabled && state.waiting_for_previous_pid.is_none()
3302        };
3303        let child = if should_spawn {
3304            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3305            let spawned = spawn_child(
3306                &spec,
3307                runtime.connection_file_path.as_deref(),
3308                self.supervisor_handle.as_ref(),
3309                &runtime.stderr_ring,
3310                runtime.capture_logs_dir.as_deref(),
3311                &runtime.child_roster,
3312                #[cfg(target_os = "linux")]
3313                runtime.cgroup_placement.as_ref(),
3314            );
3315            #[cfg(test)]
3316            self.test_after_first_spawn.run(&spec.module_id);
3317            match spawned {
3318                Ok(child) => {
3319                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3320                    if let Some(spawn) = activity_spawn {
3321                        spawn.complete(snapshot);
3322                    }
3323                    Some(child)
3324                }
3325                Err(err) => {
3326                    if runtime.health.critical {
3327                        error!(
3328                            module_id = %spec.module_id,
3329                            program = %spec.program.display(),
3330                            error = %err,
3331                            "critical configured module failed to spawn; marking failed and alerting"
3332                        );
3333                    } else {
3334                        error!(
3335                            module_id = %spec.module_id,
3336                            program = %spec.program.display(),
3337                            error = %err,
3338                            "configured module failed to spawn; marking failed and continuing"
3339                        );
3340                    }
3341                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3342                        state.state = ModuleState::Failed;
3343                        clear_current_process_facts(state);
3344                    })?;
3345                    None
3346                }
3347            }
3348        } else {
3349            None
3350        };
3351        self.start_monitor(spec, runtime, &module, child, commands);
3352        Ok(module)
3353    }
3354
3355    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3356        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3357        SupervisorRuntimeConfig {
3358            start_requester: None,
3359            pending_activity_restart: false,
3360            provenance_probe: self.provenance_probe.clone(),
3361            scheduled_respawn: Arc::default(),
3362            deferred_reload_reply: Arc::default(),
3363            restart_policy: self.restart_policy,
3364            drain_timeout: self.drain_timeout,
3365            // Shared with this module's roster copy: daemon shutdown waits on
3366            // each child for the module's own drain budget, as resolved now.
3367            child_roster: self
3368                .child_roster
3369                .for_module(Arc::clone(&effective_drain_timeout)),
3370            effective_drain_timeout,
3371            default_drain_timeout: self.drain_timeout,
3372            health: self.health.clone(),
3373            connection_file_path: self.connection_file_path.clone(),
3374            capture_logs_dir: self.capture_logs_dir.clone(),
3375            forwarding: self.forwarding.clone(),
3376            supervisor_handle: self.supervisor_handle.clone(),
3377            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3378            terminal_ring: Arc::new(Mutex::new(
3379                TerminalRing::new(
3380                    TerminalRingConfig::default(),
3381                    self.daemon_start_clock.started_at_ms(),
3382                )
3383                .with_start_clock(self.daemon_start_clock)
3384                .with_journal(self.terminal_journal.clone())
3385                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3386            )),
3387            spawn_events: self.spawn_events.clone(),
3388            #[cfg(target_os = "linux")]
3389            cgroup_placement: self.cgroup_placement.clone(),
3390            #[cfg(test)]
3391            test_seed_stale_facts_before_enable_spawn: false,
3392            #[cfg(test)]
3393            test_reload_exit_record_gate: None,
3394        }
3395    }
3396
3397    fn supervised_module(
3398        &self,
3399        spec: ModuleSpec,
3400        runtime: SupervisorRuntimeConfig,
3401        snapshot: SharedSnapshot,
3402        child: Option<SupervisedChild>,
3403    ) -> SupervisedModule {
3404        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3405        self.start_monitor(spec, runtime, &module, child, commands);
3406        module
3407    }
3408
3409    fn register_module(
3410        &self,
3411        spec: &ModuleSpec,
3412        runtime: &SupervisorRuntimeConfig,
3413        snapshot: SharedSnapshot,
3414    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3415        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3416            spec: spec.clone(),
3417            health: runtime.health.clone(),
3418        }));
3419        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3420        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3421        // The module's OWN policy, which may be its per-module config rather than
3422        // the supervisor-wide one; status must report the budget the supervise
3423        // loop actually enforces.
3424        let restart_policy = runtime.restart_policy;
3425        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3426        let (tx, rx) = mpsc::channel(4);
3427        let module_id = spec.module_id.clone();
3428        let module = SupervisedModule {
3429            inner: Arc::new(SupervisedModuleInner {
3430                module_id: module_id.clone(),
3431                registry: Arc::clone(&self.registry),
3432                snapshot,
3433                configuration,
3434                stderr_ring,
3435                terminal_ring,
3436                commands: tx,
3437                monitor: Mutex::new(None),
3438                restart_policy,
3439                effective_drain_timeout,
3440                provenance_probe: self.provenance_probe.clone(),
3441            }),
3442        };
3443        // Identity gates and the configured mark must precede the roster entry,
3444        // so a client never sees a module without its admission policy.
3445        if let Some(supervisor_handle) = &self.supervisor_handle {
3446            supervisor_handle.insert(module.clone());
3447        }
3448        (module, rx)
3449    }
3450
3451    fn start_monitor(
3452        &self,
3453        spec: ModuleSpec,
3454        runtime: SupervisorRuntimeConfig,
3455        module: &SupervisedModule,
3456        child: Option<SupervisedChild>,
3457        commands: mpsc::Receiver<SupervisorCommand>,
3458    ) {
3459        let monitor = tokio::spawn(supervise_loop(
3460            spec,
3461            runtime,
3462            Arc::clone(&self.registry),
3463            Arc::clone(&self.process_liveness),
3464            Arc::clone(&module.inner.snapshot),
3465            child,
3466            commands,
3467        ));
3468        *module
3469            .inner
3470            .monitor
3471            .lock()
3472            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3473    }
3474}
3475
3476impl Default for Supervisor {
3477    fn default() -> Self {
3478        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3479    }
3480}
3481
3482/// Handle to one supervised child process.
3483#[derive(Clone)]
3484pub struct SupervisedModule {
3485    inner: Arc<SupervisedModuleInner>,
3486}
3487
3488struct SupervisedModuleInner {
3489    module_id: String,
3490    registry: Arc<Registry>,
3491    snapshot: SharedSnapshot,
3492    configuration: Arc<Mutex<SupervisedConfiguration>>,
3493    stderr_ring: Arc<Mutex<StderrRing>>,
3494    terminal_ring: Arc<Mutex<TerminalRing>>,
3495    commands: mpsc::Sender<SupervisorCommand>,
3496    monitor: Mutex<Option<JoinHandle<()>>>,
3497    /// Copied from the supervisor's runtime config at spawn so `status()` can
3498    /// report the restart budget without reaching back into the supervisor. The
3499    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3500    restart_policy: RestartPolicy,
3501    effective_drain_timeout: Arc<Mutex<Duration>>,
3502    provenance_probe: ExecutableIdentityProbe,
3503}
3504
3505impl fmt::Debug for SupervisedModule {
3506    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3507        f.debug_struct("SupervisedModule")
3508            .field("module_id", &self.inner.module_id)
3509            .field("status", &self.status())
3510            .finish_non_exhaustive()
3511    }
3512}
3513
3514impl SupervisedModule {
3515    pub fn module_id(&self) -> &str {
3516        &self.inner.module_id
3517    }
3518
3519    /// Test-only: put one probe miss on the streak, the way
3520    /// `handle_health_probe_failure` does, so tests can assert what a later
3521    /// event does to the streak without driving the whole probe loop.
3522    #[cfg(test)]
3523    pub(crate) fn record_health_probe_failure_for_test(
3524        &self,
3525        detail: &str,
3526    ) -> Result<(), SuperviseError> {
3527        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3528            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3529            state.health.detail = Some(detail.to_string());
3530        })
3531    }
3532
3533    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3534        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3535    }
3536
3537    pub(crate) fn registration_end_reason(
3538        &self,
3539    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3540        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3541        Ok(match snapshot.state {
3542            ModuleState::Draining if snapshot.draining_to_replace => {
3543                Some(RegistrationEndReason::SupervisorRestart)
3544            }
3545            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3546            ModuleState::Restarting if snapshot.draining_to_replace => {
3547                Some(RegistrationEndReason::SupervisorRestart)
3548            }
3549            _ => None,
3550        })
3551    }
3552
3553    /// The module's retained stderr, newest lines last.
3554    ///
3555    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3556    /// module, `supervisor.list` renders every module, and putting it in the
3557    /// shared snapshot would make each status read carry a payload almost nobody
3558    /// asked for. Callers that want the text ask for it.
3559    pub fn stderr_tail(
3560        &self,
3561        max_lines: Option<usize>,
3562        max_bytes: Option<usize>,
3563    ) -> StderrTailSnapshot {
3564        self.inner
3565            .stderr_ring
3566            .lock()
3567            .unwrap_or_else(|poisoned| poisoned.into_inner())
3568            .snapshot(max_lines, max_bytes)
3569    }
3570
3571    /// The module's bounded terminal history, oldest retained exit first.
3572    ///
3573    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3574    /// daemon whose in-memory history was necessarily reset.
3575    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3576        self.inner
3577            .terminal_ring
3578            .lock()
3579            .unwrap_or_else(|poisoned| poisoned.into_inner())
3580            .snapshot()
3581    }
3582
3583    /// Retained observations from the current ring and all journal generations.
3584    ///
3585    /// Blocking: this reads the journal files. Async callers use
3586    /// [`Self::read_durable_terminal_history`].
3587    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3588        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3589    }
3590
3591    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3592    /// read (up to every retained generation) never occupies a runtime worker.
3593    /// Fails only if the blocking task could not finish (runtime shutdown or a
3594    /// panic in the read).
3595    pub(crate) async fn read_durable_terminal_history(
3596        &self,
3597    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3598        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3599        let module_id = self.inner.module_id.clone();
3600        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3601            .await
3602    }
3603
3604    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3605        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3606            .map(|(status, _)| status)
3607    }
3608
3609    pub(crate) fn record_deliberate_severance(
3610        &self,
3611        identity: ProcessIdentity,
3612    ) -> Result<bool, SuperviseError> {
3613        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3614        if snapshot.pid != Some(identity.pid)
3615            || snapshot.process_start_time != Some(identity.start_time)
3616        {
3617            return Ok(false);
3618        }
3619        snapshot.deliberate_severance = Some(identity);
3620        Ok(true)
3621    }
3622
3623    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3624    ///
3625    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3626    /// does not produce reader-observability logs.
3627    pub(crate) fn status_for_control(
3628        &self,
3629        caller: &'static str,
3630    ) -> Result<ModuleStatus, SuperviseError> {
3631        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3632            .map(|(status, _)| status)
3633    }
3634
3635    fn status_with_snapshot_lock(
3636        &self,
3637        snapshot: &SharedSnapshot,
3638        caller: Option<&'static str>,
3639    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3640        let mut guard = match caller {
3641            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3642            None => lock_snapshot(snapshot)?,
3643        };
3644        // Read the budget through the pruning path so a reader sees the same
3645        // in-window count the restart decision would use, not a stale total.
3646        let restart_count =
3647            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3648        let snapshot = guard.clone();
3649        drop(guard);
3650        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3651            SuperviseError::StatePoisoned {
3652                module_id: Some(self.inner.module_id.clone()),
3653            }
3654        })?;
3655        let registration_active = self
3656            .inner
3657            .registry
3658            .get_module(&self.inner.module_id)
3659            .map_err(SuperviseError::Registry)?
3660            .is_some();
3661        let protocol = snapshot
3662            .spawned_protocol
3663            .unwrap_or(self.declared_protocol()?);
3664        let running_process =
3665            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3666        // Registration is the difference between the two protocols and the only
3667        // one: a subc module that has not registered cannot serve a request even
3668        // though its process is up, and a `none` module never registers at all,
3669        // so requiring it there would pin `live` to false for the whole life of
3670        // a perfectly healthy process.
3671        let live = match protocol {
3672            ModuleProtocol::Subc => running_process && registration_active,
3673            ModuleProtocol::None => running_process,
3674        };
3675
3676        Ok((
3677            ModuleStatus {
3678                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3679                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3680                module_id: self.inner.module_id.clone(),
3681                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3682                state: snapshot.state,
3683                enabled: snapshot.enabled,
3684                process_alive: snapshot.process_alive,
3685                registration_active,
3686                protocol,
3687                live,
3688                restart_count,
3689                lifetime_restarts: snapshot.lifetime_restarts,
3690                spawn_generation: snapshot.spawn_generation,
3691                max_restarts: self.inner.restart_policy.max_restarts,
3692                restart_window: self.inner.restart_policy.window,
3693                drain_timeout,
3694                restart_backoff: self.inner.restart_policy.backoff,
3695                restart_max_backoff: self.inner.restart_policy.max_backoff,
3696                pid: snapshot.reported_pid(),
3697                spawned_at_ms: snapshot.spawned_at_ms,
3698                spawned_from: snapshot.spawned_from,
3699                launch_spec_digest: snapshot.launch_spec_digest,
3700                process_start_time: snapshot.process_start_time,
3701                last_exit: snapshot.last_exit,
3702                health: snapshot.health,
3703            },
3704            snapshot.spawned_file_identity.clone(),
3705        ))
3706    }
3707
3708    #[cfg(test)]
3709    pub(crate) fn hold_snapshot_for_test(
3710        &self,
3711        acquired: std::sync::mpsc::Sender<()>,
3712        hold: Duration,
3713    ) -> std::thread::JoinHandle<()> {
3714        let snapshot = Arc::clone(&self.inner.snapshot);
3715        std::thread::spawn(move || {
3716            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3717            acquired
3718                .send(())
3719                .expect("test receiver waits for snapshot lock");
3720            std::thread::sleep(hold);
3721        })
3722    }
3723
3724    /// The status and the running-image check for `supervisor.provenance`,
3725    /// taken from one status read. The exec acknowledgement can land between
3726    /// two separate reads, and the reply would then pair "no pid yet" with an
3727    /// image observed after the module started, which describes no single
3728    /// moment.
3729    pub(crate) async fn status_and_running_image_agreement(
3730        &self,
3731    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3732        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3733        let image = self
3734            .inner
3735            .provenance_probe
3736            .observe(
3737                status.pid,
3738                status.spawned_from.as_deref(),
3739                identity,
3740                status.process_start_time,
3741            )
3742            .await;
3743        Ok((status, image))
3744    }
3745
3746    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3747        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3748            Ok(snapshot) => snapshot.clone(),
3749            Err(_) => {
3750                return subc_control::RunningImageAgreement::Unavailable {
3751                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3752                };
3753            }
3754        };
3755        self.inner
3756            .provenance_probe
3757            .observe(
3758                snapshot.reported_pid(),
3759                snapshot.spawned_from.as_deref(),
3760                snapshot.spawned_file_identity.clone(),
3761                snapshot.process_start_time,
3762            )
3763            .await
3764    }
3765
3766    /// Memory and CPU time of the module's current process, read now. Only the
3767    /// process the supervisor spawned is read, not processes it has started.
3768    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3769        #[cfg(windows)]
3770        {
3771            let image = lock_snapshot(&self.inner.snapshot)
3772                .ok()
3773                .and_then(|snapshot| {
3774                    snapshot
3775                        .spawned_file_identity
3776                        .as_ref()
3777                        .and_then(|identity| identity.windows_image.clone())
3778                });
3779            match image {
3780                Some(image) => {
3781                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3782                }
3783                None => subc_control::ChildResourceUsage::Unavailable {
3784                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3785                },
3786            }
3787        }
3788        #[cfg(not(windows))]
3789        {
3790            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3791                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3792                Err(_) => {
3793                    return subc_control::ChildResourceUsage::Unavailable {
3794                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3795                    }
3796                }
3797            };
3798            crate::child_resources::read(pid, start_time)
3799        }
3800    }
3801
3802    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3803        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3804        Ok(match snapshot.state {
3805            ModuleState::Restarting => true,
3806            ModuleState::Failed | ModuleState::Disabled => false,
3807            _ if snapshot.enabled && snapshot.respawn_pending => true,
3808            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3809        })
3810    }
3811
3812    #[cfg(test)]
3813    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3814        self.is_warming_with_snapshot_lock(None)
3815    }
3816
3817    pub(crate) fn is_warming_for_control(
3818        &self,
3819        caller: &'static str,
3820    ) -> Result<bool, SuperviseError> {
3821        self.is_warming_with_snapshot_lock(Some(caller))
3822    }
3823
3824    fn is_warming_with_snapshot_lock(
3825        &self,
3826        caller: Option<&'static str>,
3827    ) -> Result<bool, SuperviseError> {
3828        let snapshot = match caller {
3829            Some(caller) => {
3830                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3831            }
3832            None => lock_snapshot(&self.inner.snapshot)?,
3833        }
3834        .clone();
3835        Ok(matches!(
3836            snapshot.state,
3837            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3838        ))
3839    }
3840
3841    /// Drain the module and stop monitoring it.
3842    pub async fn drain(&self) -> Result<(), SuperviseError> {
3843        self.stop().await
3844    }
3845
3846    pub(crate) async fn retire_requested(
3847        &self,
3848        requester: Option<Principal>,
3849    ) -> Result<(), SuperviseError> {
3850        match self.state()? {
3851            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3852            ModuleState::Starting
3853            | ModuleState::Running
3854            | ModuleState::Unresponsive
3855            | ModuleState::Restarting
3856            | ModuleState::Draining
3857            | ModuleState::Disabled => {}
3858        }
3859
3860        let (reply_tx, reply_rx) = oneshot::channel();
3861        self.inner
3862            .commands
3863            .send(SupervisorCommand::Retire {
3864                requester,
3865                reply: reply_tx,
3866            })
3867            .await
3868            .map_err(|_| SuperviseError::CommandClosed {
3869                module_id: self.inner.module_id.clone(),
3870            })?;
3871        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3872            module_id: self.inner.module_id.clone(),
3873        })?
3874    }
3875
3876    pub async fn stop(&self) -> Result<(), SuperviseError> {
3877        self.stop_requested(None).await
3878    }
3879
3880    pub(crate) async fn stop_requested(
3881        &self,
3882        requester: Option<Principal>,
3883    ) -> Result<(), SuperviseError> {
3884        match self.state()? {
3885            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3886            ModuleState::Starting
3887            | ModuleState::Running
3888            | ModuleState::Unresponsive
3889            | ModuleState::Restarting
3890            | ModuleState::Draining
3891            | ModuleState::Disabled => {}
3892        }
3893
3894        let (reply_tx, reply_rx) = oneshot::channel();
3895        self.inner
3896            .commands
3897            .send(SupervisorCommand::Drain {
3898                requester,
3899                reply: reply_tx,
3900            })
3901            .await
3902            .map_err(|_| SuperviseError::CommandClosed {
3903                module_id: self.inner.module_id.clone(),
3904            })?;
3905        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3906            module_id: self.inner.module_id.clone(),
3907        })?
3908    }
3909
3910    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3911        self.restart_requested(drain_timeout_ms, None).await
3912    }
3913
3914    pub(crate) async fn restart_requested(
3915        &self,
3916        drain_timeout_ms: Option<u64>,
3917        requester: Option<Principal>,
3918    ) -> Result<(), SuperviseError> {
3919        let queued_at = Instant::now();
3920        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3921        let (reply_tx, reply_rx) = oneshot::channel();
3922        self.inner
3923            .commands
3924            .send(SupervisorCommand::Restart {
3925                requester,
3926                drain_timeout_ms,
3927                received_at_generation,
3928                queued_at,
3929                reply: reply_tx,
3930            })
3931            .await
3932            .map_err(|_| SuperviseError::CommandClosed {
3933                module_id: self.inner.module_id.clone(),
3934            })?;
3935        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3936            module_id: self.inner.module_id.clone(),
3937        })?
3938    }
3939
3940    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3941    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3942    /// process then drains in the background of the supervise loop) or has
3943    /// failed, leaving the old process serving.
3944    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3945        self.swap_requested(ready_timeout, None).await
3946    }
3947
3948    pub(crate) async fn swap_requested(
3949        &self,
3950        ready_timeout: Option<Duration>,
3951        requester: Option<Principal>,
3952    ) -> Result<(), SuperviseError> {
3953        let (reply_tx, reply_rx) = oneshot::channel();
3954        self.inner
3955            .commands
3956            .send(SupervisorCommand::Swap {
3957                requester,
3958                ready_timeout,
3959                reply: reply_tx,
3960            })
3961            .await
3962            .map_err(|_| SuperviseError::CommandClosed {
3963                module_id: self.inner.module_id.clone(),
3964            })?;
3965        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3966            module_id: self.inner.module_id.clone(),
3967        })?
3968    }
3969
3970    pub async fn reload(&self) -> Result<(), SuperviseError> {
3971        self.reload_requested(None).await
3972    }
3973
3974    pub(crate) async fn reload_requested(
3975        &self,
3976        requester: Option<Principal>,
3977    ) -> Result<(), SuperviseError> {
3978        let (reply_tx, reply_rx) = oneshot::channel();
3979        self.inner
3980            .commands
3981            .send(SupervisorCommand::Reload {
3982                requester,
3983                reply: reply_tx,
3984            })
3985            .await
3986            .map_err(|_| SuperviseError::CommandClosed {
3987                module_id: self.inner.module_id.clone(),
3988            })?;
3989        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3990            module_id: self.inner.module_id.clone(),
3991        })?
3992    }
3993
3994    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3995        self.set_enabled_requested(enabled, None).await
3996    }
3997
3998    pub(crate) async fn set_enabled_requested(
3999        &self,
4000        enabled: bool,
4001        requester: Option<Principal>,
4002    ) -> Result<bool, SuperviseError> {
4003        let (reply_tx, reply_rx) = oneshot::channel();
4004        self.inner
4005            .commands
4006            .send(SupervisorCommand::SetEnabled {
4007                requester,
4008                enabled,
4009                reply: reply_tx,
4010            })
4011            .await
4012            .map_err(|_| SuperviseError::CommandClosed {
4013                module_id: self.inner.module_id.clone(),
4014            })?;
4015        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4016            module_id: self.inner.module_id.clone(),
4017        })?
4018    }
4019
4020    /// The current process's protocol, or the configured protocol when down.
4021    /// A rescan stores the next launch spec without changing how an existing
4022    /// process registers, serves routes, is probed, or exits.
4023    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4024        let configured = self
4025            .inner
4026            .configuration
4027            .lock()
4028            .map_err(|_| SuperviseError::StatePoisoned {
4029                module_id: Some(self.inner.module_id.clone()),
4030            })?
4031            .spec
4032            .protocol;
4033        let state = lock_snapshot(&self.inner.snapshot)?;
4034        Ok(state.spawned_protocol.unwrap_or(configured))
4035    }
4036
4037    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4038        let configuration =
4039            self.inner
4040                .configuration
4041                .lock()
4042                .map_err(|_| SuperviseError::StatePoisoned {
4043                    module_id: Some(self.inner.module_id.clone()),
4044                })?;
4045        Ok((configuration.spec.clone(), configuration.health.clone()))
4046    }
4047
4048    /// Replace this module's launch spec, keeping its health and drain policy,
4049    /// the way a rescan does for a changed config entry. The running process is
4050    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4051    #[cfg(any(test, feature = "test-support"))]
4052    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4053        let (_, health) = self.configuration()?;
4054        let drain_timeout_ms = u64::try_from(
4055            self.inner
4056                .effective_drain_timeout
4057                .lock()
4058                .unwrap_or_else(|poisoned| poisoned.into_inner())
4059                .as_millis(),
4060        )
4061        .ok();
4062        self.update_configuration(spec, health, drain_timeout_ms)
4063            .await
4064    }
4065
4066    pub(crate) async fn update_configuration(
4067        &self,
4068        spec: ModuleSpec,
4069        health: HealthConfig,
4070        drain_timeout_ms: Option<u64>,
4071    ) -> Result<(), SuperviseError> {
4072        if spec.module_id != self.inner.module_id {
4073            return Err(SuperviseError::InvalidSpec {
4074                reason: "a supervised module's module_id cannot be changed".to_string(),
4075            });
4076        }
4077        validate_spec(&spec)?;
4078        let (reply_tx, reply_rx) = oneshot::channel();
4079        self.inner
4080            .commands
4081            .send(SupervisorCommand::UpdateConfiguration {
4082                spec: spec.clone(),
4083                health: health.clone(),
4084                drain_timeout_ms,
4085                reply: reply_tx,
4086            })
4087            .await
4088            .map_err(|_| SuperviseError::CommandClosed {
4089                module_id: self.inner.module_id.clone(),
4090            })?;
4091        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4092            module_id: self.inner.module_id.clone(),
4093        })?;
4094        let mut configuration =
4095            self.inner
4096                .configuration
4097                .lock()
4098                .map_err(|_| SuperviseError::StatePoisoned {
4099                    module_id: Some(self.inner.module_id.clone()),
4100                })?;
4101        configuration.spec = spec;
4102        configuration.health = health;
4103        Ok(())
4104    }
4105}
4106
4107impl Drop for SupervisedModuleInner {
4108    fn drop(&mut self) {
4109        let Ok(mut monitor) = self.monitor.lock() else {
4110            return;
4111        };
4112        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4113            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4114                state.state = ModuleState::Stopped;
4115                clear_current_process_facts(state);
4116            });
4117            monitor.abort();
4118        }
4119        let _ = monitor.take();
4120    }
4121}
4122
4123#[derive(Debug)]
4124enum SupervisorCommand {
4125    Drain {
4126        requester: Option<Principal>,
4127        reply: oneshot::Sender<Result<(), SuperviseError>>,
4128    },
4129    Retire {
4130        requester: Option<Principal>,
4131        reply: oneshot::Sender<Result<(), SuperviseError>>,
4132    },
4133    Restart {
4134        requester: Option<Principal>,
4135        /// Operator override for this one restart's drain budget, in ms. `None`
4136        /// uses the module's configured/default budget; `Some(0)` cuts
4137        /// immediately (wedge bounce: a stuck request never settles, so
4138        /// waiting only delays recovery).
4139        drain_timeout_ms: Option<u64>,
4140        /// The module's `spawn_generation` when the request was received, before
4141        /// it waited in the command queue. A queued restart whose module has
4142        /// since spawned a newer process is already satisfied (see the handler).
4143        received_at_generation: u64,
4144        /// When the request entered the command queue, so the handler can log
4145        /// how long it waited behind the loop's other work.
4146        queued_at: Instant,
4147        reply: oneshot::Sender<Result<(), SuperviseError>>,
4148    },
4149    Reload {
4150        requester: Option<Principal>,
4151        reply: oneshot::Sender<Result<(), SuperviseError>>,
4152    },
4153    SetEnabled {
4154        requester: Option<Principal>,
4155        enabled: bool,
4156        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4157    },
4158    UpdateConfiguration {
4159        spec: ModuleSpec,
4160        health: HealthConfig,
4161        /// Per-module drain override from the new config; `None` re-resolves to
4162        /// the supervisor-wide default.
4163        drain_timeout_ms: Option<u64>,
4164        reply: oneshot::Sender<()>,
4165    },
4166    Swap {
4167        requester: Option<Principal>,
4168        /// How long the candidate may take to register and declare itself
4169        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4170        ready_timeout: Option<Duration>,
4171        /// Answered at cutover or failure; the incumbent's drain follows.
4172        reply: oneshot::Sender<Result<(), SuperviseError>>,
4173    },
4174}
4175
4176#[derive(Debug)]
4177pub enum SuperviseError {
4178    InvalidSpec {
4179        reason: String,
4180    },
4181    Spawn {
4182        program: PathBuf,
4183        source: io::Error,
4184        cgroup_path: Option<PathBuf>,
4185    },
4186    Cgroup {
4187        module_id: String,
4188        source: io::Error,
4189    },
4190    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4191    /// than spawn a reserved module without its identity binding.
4192    LaunchNonce {
4193        reason: String,
4194    },
4195    Wait {
4196        module_id: String,
4197        source: io::Error,
4198    },
4199    Kill {
4200        module_id: String,
4201        source: io::Error,
4202    },
4203    Forwarding(ForwardingError),
4204    Registry(RegistryError),
4205    ReloadUnavailable {
4206        module_id: String,
4207        reason: String,
4208    },
4209    /// An operator restart/reload was requested for a module that is currently
4210    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4211    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4212    /// by a restart, so these commands are rejected instead of re-enabling it.
4213    Disabled {
4214        module_id: String,
4215    },
4216    ReloadFailed {
4217        module_id: String,
4218        reason: String,
4219    },
4220    RegistrationStillActive {
4221        module_id: String,
4222        waited: Duration,
4223    },
4224    StatePoisoned {
4225        module_id: Option<String>,
4226    },
4227    CommandClosed {
4228        module_id: String,
4229    },
4230    /// A restart or reload arrived while a swap's candidate was warming. The
4231    /// swap owns the module until it cuts over or fails; a stop or disable
4232    /// would have aborted it instead.
4233    SwapInProgress {
4234        module_id: String,
4235    },
4236    /// A swap was refused before anything was spawned.
4237    SwapRefused {
4238        module_id: String,
4239        reason: SwapRefusal,
4240    },
4241    /// A swap spawned a candidate and gave up on it. The candidate has been
4242    /// killed and its slot freed; the incumbent was left serving and was never
4243    /// drained, except in the one `CutoverLost` case described on that arm.
4244    SwapFailed {
4245        module_id: String,
4246        arm: SwapFailureArm,
4247        detail: String,
4248        /// How the candidate exited, when it exited on its own before the
4249        /// supervisor gave up on it.
4250        candidate_exit: Option<ExitReport>,
4251    },
4252}
4253
4254/// Why a swap was refused before a candidate was spawned.
4255#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4256pub enum SwapRefusal {
4257    /// The module's config does not declare `overlap: "safe"`.
4258    OverlapExclusive,
4259    /// The module is not registered, so there is no incumbent to keep serving
4260    /// and nothing a swap would improve on; a plain restart is the tool.
4261    NotRegistered,
4262    /// The module does not speak the subc wire, so a candidate could never
4263    /// register or declare itself ready.
4264    ProtocolNone,
4265    /// The supervisor lacks the forwarding table (to cut routes over) or the
4266    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4267    NotConfigured,
4268    /// A swap is already open for this module.
4269    AlreadySwapping,
4270}
4271
4272impl SwapRefusal {
4273    pub fn as_str(self) -> &'static str {
4274        match self {
4275            Self::OverlapExclusive => "overlap_exclusive",
4276            Self::NotRegistered => "not_registered",
4277            Self::ProtocolNone => "protocol_none",
4278            Self::NotConfigured => "not_configured",
4279            Self::AlreadySwapping => "already_swapping",
4280        }
4281    }
4282}
4283
4284/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4285/// serving and undrained; see `CutoverLost`.
4286#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4287pub enum SwapFailureArm {
4288    /// The candidate process could not be started.
4289    SpawnFailed,
4290    /// The candidate did not register within the readiness budget.
4291    NeverRegistered,
4292    /// The candidate registered but did not declare itself ready in time.
4293    NeverReady,
4294    /// The candidate exited before cutover.
4295    CandidateExited,
4296    /// The candidate declared itself ready but failed its health probe.
4297    CandidateUnhealthy,
4298    /// An operator stop, disable or retire arrived while the candidate warmed.
4299    /// The candidate was killed and the operator's command then carried out on
4300    /// the incumbent.
4301    Interrupted,
4302    /// The candidate's connection closed at the moment of cutover. If it
4303    /// closed before forwarding moved, the incumbent is untouched. If it closed
4304    /// between the forwarding and registry halves of cutover, forwarding can no
4305    /// longer route to the incumbent, so the module is restarted plainly.
4306    CutoverLost,
4307}
4308
4309impl SwapFailureArm {
4310    pub fn as_str(self) -> &'static str {
4311        match self {
4312            Self::SpawnFailed => "spawn_failed",
4313            Self::NeverRegistered => "never_registered",
4314            Self::NeverReady => "never_ready",
4315            Self::CandidateExited => "candidate_exited",
4316            Self::CandidateUnhealthy => "candidate_unhealthy",
4317            Self::Interrupted => "interrupted",
4318            Self::CutoverLost => "cutover_lost",
4319        }
4320    }
4321}
4322
4323impl fmt::Display for SuperviseError {
4324    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4325        match self {
4326            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4327            Self::Spawn {
4328                program,
4329                source,
4330                cgroup_path: Some(cgroup_path),
4331            } => write!(
4332                f,
4333                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4334                cgroup_path.display(),
4335                program.display()
4336            ),
4337            Self::Spawn {
4338                program,
4339                source,
4340                cgroup_path: None,
4341            } => write!(
4342                f,
4343                "failed to spawn module '{}': {source}",
4344                program.display()
4345            ),
4346            Self::Cgroup { module_id, source } => {
4347                write!(
4348                    f,
4349                    "failed to prepare cgroup for module '{module_id}': {source}"
4350                )
4351            }
4352            Self::LaunchNonce { reason } => {
4353                write!(
4354                    f,
4355                    "failed to generate reserved-module launch nonce: {reason}"
4356                )
4357            }
4358            Self::Wait { module_id, source } => {
4359                write!(f, "failed to wait for module '{module_id}': {source}")
4360            }
4361            Self::Kill { module_id, source } => {
4362                write!(f, "failed to kill module '{module_id}': {source}")
4363            }
4364            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4365            Self::Registry(err) => write!(f, "registry error: {err}"),
4366            Self::ReloadUnavailable { module_id, reason } => {
4367                write!(f, "reload unavailable for module '{module_id}': {reason}")
4368            }
4369            Self::Disabled { module_id } => {
4370                write!(
4371                    f,
4372                    "module '{module_id}' is disabled; enable it before restart or reload"
4373                )
4374            }
4375            Self::ReloadFailed { module_id, reason } => {
4376                write!(f, "reload failed for module '{module_id}': {reason}")
4377            }
4378            Self::RegistrationStillActive { module_id, waited } => write!(
4379                f,
4380                "module '{module_id}' registration remained active after waiting {waited:?}"
4381            ),
4382            Self::StatePoisoned { module_id } => match module_id {
4383                Some(module_id) => {
4384                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4385                }
4386                None => write!(f, "supervisor state was poisoned"),
4387            },
4388            Self::CommandClosed { module_id } => {
4389                write!(
4390                    f,
4391                    "supervisor command channel for module '{module_id}' is closed"
4392                )
4393            }
4394            Self::SwapInProgress { module_id } => write!(
4395                f,
4396                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4397            ),
4398            Self::SwapRefused { module_id, reason } => match reason {
4399                SwapRefusal::OverlapExclusive => write!(
4400                    f,
4401                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4402                ),
4403                SwapRefusal::NotRegistered => write!(
4404                    f,
4405                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4406                ),
4407                SwapRefusal::ProtocolNone => write!(
4408                    f,
4409                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4410                ),
4411                SwapRefusal::NotConfigured => write!(
4412                    f,
4413                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4414                ),
4415                SwapRefusal::AlreadySwapping => {
4416                    write!(f, "module '{module_id}' is already being swapped")
4417                }
4418            },
4419            Self::SwapFailed {
4420                module_id,
4421                arm,
4422                detail,
4423                ..
4424            } => write!(
4425                f,
4426                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4427                arm.as_str()
4428            ),
4429        }
4430    }
4431}
4432
4433impl Error for SuperviseError {
4434    fn source(&self) -> Option<&(dyn Error + 'static)> {
4435        match self {
4436            Self::Spawn { source, .. }
4437            | Self::Cgroup { source, .. }
4438            | Self::Wait { source, .. }
4439            | Self::Kill { source, .. } => Some(source),
4440            Self::Forwarding(err) => Some(err),
4441            Self::Registry(err) => Some(err),
4442            Self::LaunchNonce { .. }
4443            | Self::InvalidSpec { .. }
4444            | Self::ReloadUnavailable { .. }
4445            | Self::Disabled { .. }
4446            | Self::ReloadFailed { .. }
4447            | Self::RegistrationStillActive { .. }
4448            | Self::StatePoisoned { .. }
4449            | Self::CommandClosed { .. }
4450            | Self::SwapInProgress { .. }
4451            | Self::SwapRefused { .. }
4452            | Self::SwapFailed { .. } => None,
4453        }
4454    }
4455}
4456
4457pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4458    if spec.module_id.trim().is_empty() {
4459        return Err(SuperviseError::InvalidSpec {
4460            reason: "module_id must not be empty".to_string(),
4461        });
4462    }
4463
4464    Ok(())
4465}
4466
4467#[derive(Debug, Default)]
4468struct HealthProbeRuntime {
4469    spawn_generation: Option<u64>,
4470    configured_health: Option<HealthConfig>,
4471    registered_connection: Option<crate::ConnectionId>,
4472    advertised: bool,
4473    next_probe_at: Option<Instant>,
4474    probe_index: u64,
4475}
4476
4477fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4478    lock_snapshot(snapshot)
4479        .ok()
4480        .and_then(|state| state.spawned_protocol)
4481        .unwrap_or(spec.protocol)
4482}
4483
4484impl HealthProbeRuntime {
4485    fn refresh_registration(
4486        &mut self,
4487        spec: &ModuleSpec,
4488        runtime: &SupervisorRuntimeConfig,
4489        registry: &Registry,
4490        snapshot: &SharedSnapshot,
4491    ) {
4492        let generation = lock_snapshot(snapshot)
4493            .ok()
4494            .map(|state| state.spawn_generation);
4495        if self.spawn_generation != generation {
4496            // A replacement cannot reuse a deadline that is already due for its
4497            // predecessor. HTTP children get a full cadence to start; wire
4498            // children still wait for their own health-advertising HELLO.
4499            *self = Self {
4500                spawn_generation: generation,
4501                ..Self::default()
4502            };
4503        }
4504        if self.configured_health.as_ref() != Some(&runtime.health) {
4505            self.configured_health = Some(runtime.health.clone());
4506            self.next_probe_at = None;
4507            self.registered_connection = None;
4508            self.probe_index = 0;
4509        }
4510        // A non-wire process never registers. Only an explicitly configured
4511        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4512        // health failure for that kind of process.
4513        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4514            self.registered_connection = None;
4515            self.advertised = runtime.health.http.is_some();
4516            if !self.advertised {
4517                self.next_probe_at = None;
4518                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4519                    let unknown = ModuleHealthStatus::default();
4520                    if state.health != unknown {
4521                        state.health = unknown;
4522                    }
4523                });
4524            } else if self.next_probe_at.is_none() {
4525                self.next_probe_at = Some(
4526                    Instant::now()
4527                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4528                );
4529            }
4530            return;
4531        }
4532
4533        let registration = match registry.get_module(&spec.module_id) {
4534            Ok(registration) => registration,
4535            Err(err) => {
4536                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4537                self.advertised = false;
4538                self.next_probe_at = None;
4539                return;
4540            }
4541        };
4542
4543        let Some(registration) = registration else {
4544            self.registered_connection = None;
4545            self.advertised = false;
4546            self.next_probe_at = None;
4547            return;
4548        };
4549
4550        let advertised = registration
4551            .control_ops
4552            .iter()
4553            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4554        if !advertised {
4555            self.registered_connection = Some(registration.connection_id);
4556            self.advertised = false;
4557            self.next_probe_at = None;
4558            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4559                if state.health.status != SupervisorHealthStatus::Unknown
4560                    || state.health.consecutive_failures != 0
4561                    || state.health.last_probe_ms.is_some()
4562                    || state.health.detail.is_some()
4563                    || state.health.metrics.is_some()
4564                {
4565                    state.health.status = SupervisorHealthStatus::Unknown;
4566                    state.health.consecutive_failures = 0;
4567                    state.health.last_probe_ms = None;
4568                    state.health.detail = None;
4569                    state.health.metrics = None;
4570                }
4571            });
4572            return;
4573        }
4574
4575        let reregistered = self.registered_connection != Some(registration.connection_id);
4576        self.registered_connection = Some(registration.connection_id);
4577        self.advertised = true;
4578        if reregistered || self.next_probe_at.is_none() {
4579            self.probe_index = 0;
4580            self.next_probe_at = Some(
4581                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4582            );
4583            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4584                state.health.status = SupervisorHealthStatus::Unknown;
4585                state.health.consecutive_failures = 0;
4586                state.health.detail = None;
4587                state.health.metrics = None;
4588            });
4589        }
4590    }
4591
4592    fn wake_after(&self) -> Option<Duration> {
4593        if !self.advertised {
4594            return None;
4595        }
4596        self.next_probe_at
4597            .map(|next| next.saturating_duration_since(Instant::now()))
4598    }
4599
4600    fn due(&self) -> bool {
4601        self.advertised
4602            && self
4603                .next_probe_at
4604                .is_some_and(|next| Instant::now() >= next)
4605    }
4606
4607    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4608        self.probe_index = self.probe_index.wrapping_add(1);
4609        self.next_probe_at = Some(
4610            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4611        );
4612    }
4613}
4614
4615/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4616///
4617/// This was a struct with a single `message: String`, and every one of the
4618/// fifteen construction sites collapsed into it. Each site knows exactly what it
4619/// saw -- the lane is gone, the module did not answer in time, the module
4620/// answered with the wrong thing -- and `handle_health_probe_failure` then
4621/// treated all of them identically: increment a counter, compare to a threshold,
4622/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4623/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4624///
4625/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4626///
4627/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4628///   answer on it again.
4629/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4630///   AND with a perfectly healthy one that lost a CPU race -- which is what
4631///   happens under machine load, and is how this supervisor killed a healthy
4632///   module three times in one day.
4633/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4634///   Restarting on it is defensible, but it is not the silence case and should
4635///   never be counted as one.
4636/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4637///   anything, so it cannot be evidence about the module at all.
4638///
4639/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4640/// one that fires most often, and while every variant collapsed into one string
4641/// it carried the same weight as the strongest.
4642///
4643/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4644/// DESIGN and a reader stopping at it gets the build backwards: the restart
4645/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4646/// probes still increment the failure streak and drive escalation at the
4647/// threshold (see `is_proof_of_death` below for why that is deliberate and
4648/// what gates the change). Absence of evidence restarts modules today.
4649#[derive(Debug)]
4650enum HealthProbeEvidence {
4651    /// The module's control lane is gone. Proof of death.
4652    LaneDead,
4653    /// No reply within the deadline. Proves nothing about the module's state.
4654    NoAnswer,
4655    /// The module replied, but not with a usable health report. Proves it is alive.
4656    BadAnswer,
4657    /// The daemon could not ask. Says nothing about the module.
4658    Misconfigured,
4659}
4660
4661#[derive(Debug)]
4662struct HealthProbeError {
4663    evidence: HealthProbeEvidence,
4664    message: String,
4665}
4666
4667impl HealthProbeError {
4668    fn lane_dead(message: impl Into<String>) -> Self {
4669        Self::with(HealthProbeEvidence::LaneDead, message)
4670    }
4671
4672    fn no_answer(message: impl Into<String>) -> Self {
4673        Self::with(HealthProbeEvidence::NoAnswer, message)
4674    }
4675
4676    fn bad_answer(message: impl Into<String>) -> Self {
4677        Self::with(HealthProbeEvidence::BadAnswer, message)
4678    }
4679
4680    fn misconfigured(message: impl Into<String>) -> Self {
4681        Self::with(HealthProbeEvidence::Misconfigured, message)
4682    }
4683
4684    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4685        Self {
4686            evidence,
4687            message: message.into(),
4688        }
4689    }
4690
4691    /// Whether this observation is proof the module cannot serve.
4692    ///
4693    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4694    /// variant that fires under CPU starvation, and treating it as proof is the
4695    /// defect this enum exists to make impossible to reintroduce silently.
4696    ///
4697    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4698    /// to restart also needs a bound for the case it excludes -- a genuinely
4699    /// wedged module, alive but never answering -- and that bound must come from
4700    /// the distribution of real late-answer latencies, which nothing measures
4701    /// yet. Landing the classification first makes the later change a one-line
4702    /// decision against evidence that already exists, rather than two unproven
4703    /// changes at once.
4704    #[allow(dead_code)]
4705    fn is_proof_of_death(&self) -> bool {
4706        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4707    }
4708
4709    /// Short stable label for logs and the health snapshot.
4710    ///
4711    /// An operator reading `ck health` currently cannot tell "the module is gone"
4712    /// from "the module did not answer in five seconds", because both render as
4713    /// prose in the same field. These labels are what make the two
4714    /// distinguishable at a glance, and they are what a later restart-policy
4715    /// change will be argued from.
4716    fn label(&self) -> &'static str {
4717        match self.evidence {
4718            HealthProbeEvidence::LaneDead => "lane-dead",
4719            HealthProbeEvidence::NoAnswer => "no-answer",
4720            HealthProbeEvidence::BadAnswer => "bad-answer",
4721            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4722        }
4723    }
4724}
4725
4726impl fmt::Display for HealthProbeError {
4727    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4728        f.write_str(&self.message)
4729    }
4730}
4731
4732async fn run_health_probe_cycle(
4733    spec: &ModuleSpec,
4734    runtime: &SupervisorRuntimeConfig,
4735    registry: &Registry,
4736    process_liveness: &SupervisorProcessLiveness,
4737    snapshot: &SharedSnapshot,
4738    child: &mut Option<SupervisedChild>,
4739) {
4740    let now_ms = unix_ms_now();
4741    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4742        .then_some(runtime.health.http.as_deref())
4743        .flatten();
4744    let result = match http {
4745        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4746        None => probe_module_health(&spec.module_id, runtime, None).await,
4747    };
4748    match result {
4749        Ok(report) => {
4750            handle_health_report(
4751                spec,
4752                runtime,
4753                registry,
4754                process_liveness,
4755                snapshot,
4756                child,
4757                report,
4758                now_ms,
4759            )
4760            .await;
4761        }
4762        Err(err) => {
4763            if http.is_some() {
4764                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4765                    state.health.status = SupervisorHealthStatus::Failing;
4766                });
4767            }
4768            handle_health_probe_failure(
4769                spec,
4770                runtime,
4771                registry,
4772                process_liveness,
4773                snapshot,
4774                child,
4775                err,
4776                now_ms,
4777            )
4778            .await;
4779        }
4780    }
4781}
4782
4783pub(crate) struct HttpProbeTarget<'a> {
4784    address: std::net::SocketAddr,
4785    localhost: bool,
4786    authority: &'a str,
4787    path: String,
4788}
4789
4790/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4791/// or TLS. A URL cannot turn a local health check into an outbound connection.
4792pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4793    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4794        return Err("must not contain whitespace, controls, or a fragment".into());
4795    }
4796    let rest = url
4797        .strip_prefix("http://")
4798        .ok_or("must use plain http://")?;
4799    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4800    let (authority, suffix) = rest.split_at(split);
4801    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4802        ("::1", rest)
4803    } else {
4804        let split = authority.find(':').unwrap_or(authority.len());
4805        authority.split_at(split)
4806    };
4807    let ip: std::net::IpAddr = match host {
4808        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4809        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4810        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4811    };
4812    let port = if port.is_empty() {
4813        80
4814    } else {
4815        port.strip_prefix(':')
4816            .and_then(|p| p.parse::<u16>().ok())
4817            .filter(|p| *p > 0)
4818            .ok_or("must have a valid nonzero TCP port")?
4819    };
4820    let path = if suffix.is_empty() {
4821        "/".into()
4822    } else if suffix.starts_with('?') {
4823        format!("/{suffix}")
4824    } else {
4825        suffix.into()
4826    };
4827    Ok(HttpProbeTarget {
4828        address: std::net::SocketAddr::new(ip, port),
4829        localhost: host == "localhost",
4830        authority,
4831        path,
4832    })
4833}
4834
4835async fn probe_http_health(
4836    url: &str,
4837    deadline: Duration,
4838) -> Result<HealthReport, HealthProbeError> {
4839    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4840    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4841    // Keep partial diagnostics outside the timed future so cancellation does
4842    // not discard a status line or body bytes already received.
4843    let mut response_status = String::new();
4844    let mut body = Vec::new();
4845    let probe = async {
4846        // Resolve localhost ourselves so a hosts-file override cannot turn
4847        // this into an outbound request, while IPv6-only local servers work.
4848        let connection = match tokio::net::TcpStream::connect(target.address).await {
4849            Err(_) if target.localhost => {
4850                tokio::net::TcpStream::connect((
4851                    std::net::Ipv6Addr::LOCALHOST,
4852                    target.address.port(),
4853                ))
4854                .await
4855            }
4856            result => result,
4857        };
4858        let mut stream = connection.map_err(|error| {
4859            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4860        })?;
4861        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4862            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4863        let mut reader = BufReader::new(stream);
4864        let mut budget = 16 * 1024;
4865        let status = http_line(&mut reader, &mut budget).await?;
4866        let mut words = status.split_ascii_whitespace();
4867        let version = words.next();
4868        let code = words
4869            .next()
4870            .filter(|word| word.len() == 3)
4871            .and_then(|word| word.parse::<u16>().ok());
4872        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4873            || !code.is_some_and(|code| (100..600).contains(&code))
4874        {
4875            return Err(HealthProbeError::bad_answer(format!(
4876                "invalid HTTP status: {status}"
4877            )));
4878        }
4879        let code = code.expect("validated status code");
4880        response_status = status.clone();
4881        let mut length = None;
4882        let mut chunked = false;
4883        loop {
4884            let line = http_line(&mut reader, &mut budget).await?;
4885            if line.is_empty() {
4886                break;
4887            }
4888            if let Some((name, value)) = line.split_once(':') {
4889                if name.eq_ignore_ascii_case("content-length") {
4890                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4891                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4892                    })?);
4893                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4894                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4895                }
4896            }
4897        }
4898        if chunked {
4899            while body.len() < 200 {
4900                let line = http_line(&mut reader, &mut budget).await?;
4901                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4902                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4903                if size == 0 {
4904                    break;
4905                }
4906                let count = size.min((200 - body.len()) as u64) as usize;
4907                let start = body.len();
4908                (&mut reader)
4909                    .take(count as u64)
4910                    .read_to_end(&mut body)
4911                    .await
4912                    .map_err(|error| {
4913                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4914                    })?;
4915                if body.len() - start != count {
4916                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4917                }
4918                if size > count as u64 || body.len() == 200 {
4919                    break;
4920                }
4921                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4922                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4923                }
4924            }
4925        } else {
4926            reader
4927                .take(length.unwrap_or(200).min(200))
4928                .read_to_end(&mut body)
4929                .await
4930                .map_err(|error| {
4931                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4932                })?;
4933        }
4934        if (200..300).contains(&code) {
4935            Ok(HealthReport::ok())
4936        } else {
4937            Err(HealthProbeError::bad_answer(
4938                "HTTP health endpoint returned non-2xx",
4939            ))
4940        }
4941    };
4942    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4943        Err(HealthProbeError::no_answer(format!(
4944            "HTTP probe timed out after {deadline:?}"
4945        )))
4946    });
4947    if let Err(error) = &mut result {
4948        if !response_status.is_empty() {
4949            error.message = format!(
4950                "{}; {response_status}: {}",
4951                error.message,
4952                String::from_utf8_lossy(&body)
4953            );
4954        }
4955    }
4956    result
4957}
4958
4959async fn http_line(
4960    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4961    remaining: &mut usize,
4962) -> Result<String, HealthProbeError> {
4963    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4964    let mut line = Vec::new();
4965    (&mut *reader)
4966        .take(*remaining as u64)
4967        .read_until(b'\n', &mut line)
4968        .await
4969        .map_err(|error| {
4970            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4971        })?;
4972    *remaining -= line.len();
4973    if !line.ends_with(b"\r\n") {
4974        return Err(HealthProbeError::bad_answer(
4975            "HTTP headers are incomplete or exceed 16 KiB",
4976        ));
4977    }
4978    line.truncate(line.len() - 2);
4979    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4980}
4981
4982async fn probe_module_health(
4983    module_id: &str,
4984    runtime: &SupervisorRuntimeConfig,
4985    drain_deadline: Option<Instant>,
4986) -> Result<HealthReport, HealthProbeError> {
4987    let Some(forwarding) = runtime.forwarding.as_ref() else {
4988        return Err(HealthProbeError::misconfigured(
4989            "supervisor was not configured with a forwarding table",
4990        ));
4991    };
4992    let probe_started_at = Instant::now();
4993    let mut deadline = probe_started_at + runtime.health.deadline;
4994    if let Some(drain_deadline) = drain_deadline {
4995        deadline = deadline.min(drain_deadline);
4996    }
4997    let pending = if drain_deadline.is_some() {
4998        forwarding.begin_drain_health_probe_rpc_for(
4999            module_id,
5000            MODULE_CONTROL_OP_HEALTH_CHECK,
5001            probe_started_at,
5002            deadline,
5003        )
5004    } else {
5005        forwarding.begin_health_probe_rpc_for(
5006            module_id,
5007            MODULE_CONTROL_OP_HEALTH_CHECK,
5008            probe_started_at,
5009            deadline,
5010        )
5011    }
5012    .map_err(|err| {
5013        // The endpoint is not registered, so there is no live control lane to
5014        // ask. That is the module being absent, not slow.
5015        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5016    })?;
5017    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5018}
5019
5020/// [`probe_module_health`] for one endpoint rather than the id's active one.
5021///
5022/// A swap probes two processes that no by-id lookup reaches: its candidate
5023/// before cutover, and its superseded incumbent (for busy gauges) while the
5024/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5025/// bounds the by-id drain probe.
5026async fn probe_endpoint_health(
5027    endpoint: crate::ModuleEndpointId,
5028    runtime: &SupervisorRuntimeConfig,
5029    deadline_cap: Option<Instant>,
5030) -> Result<HealthReport, HealthProbeError> {
5031    let Some(forwarding) = runtime.forwarding.as_ref() else {
5032        return Err(HealthProbeError::misconfigured(
5033            "supervisor was not configured with a forwarding table",
5034        ));
5035    };
5036    let probe_started_at = Instant::now();
5037    let mut deadline = probe_started_at + runtime.health.deadline;
5038    if let Some(cap) = deadline_cap {
5039        deadline = deadline.min(cap);
5040    }
5041    let pending = forwarding
5042        .begin_endpoint_health_probe_rpc_for(
5043            endpoint,
5044            MODULE_CONTROL_OP_HEALTH_CHECK,
5045            probe_started_at,
5046            deadline,
5047        )
5048        .map_err(|err| {
5049            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5050        })?;
5051    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5052}
5053
5054/// Send a begun health probe and classify its answer.
5055async fn await_health_probe(
5056    forwarding: &ForwardingTable,
5057    pending: PendingModuleControlRpc,
5058    deadline: Instant,
5059    probe_budget: Duration,
5060) -> Result<HealthReport, HealthProbeError> {
5061    let PendingModuleControlRpc {
5062        endpoint,
5063        module_sink,
5064        negotiated_ver,
5065        corr,
5066        receiver,
5067    } = pending;
5068    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5069        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5070    })?;
5071    let frame = Frame::build_with_version(
5072        negotiated_ver,
5073        FrameType::Request,
5074        control_flags(),
5075        0,
5076        0,
5077        corr,
5078        body,
5079    )
5080    .map_err(|err| {
5081        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5082    })?;
5083
5084    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5085    // blocks waiting for capacity when the module's egress queue is full, and an
5086    // unbounded await here freezes the whole supervision actor (it stops polling
5087    // Child::wait and supervisor commands), making the module unrecoverable
5088    // in-band. On timeout the probe fails like any transport failure.
5089    match timeout_at(deadline, module_sink.send(frame)).await {
5090        Ok(Ok(())) => {}
5091        Ok(Err(err)) => {
5092            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5093            // A closed sink means the module's egress channel is gone -- the
5094            // receiving half is dropped when its connection tears down. Proof.
5095            return Err(HealthProbeError::lane_dead(format!(
5096                "failed to send health.check: {err}"
5097            )));
5098        }
5099        Err(_elapsed) => {
5100            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5101            // A full egress queue means the module is not draining its socket, which
5102            // is consistent with a wedged module AND with one whose reader is merely
5103            // starved. Silence, not proof.
5104            return Err(HealthProbeError::no_answer(
5105                "health.check send timed out before enqueue (module egress full)",
5106            ));
5107        }
5108    }
5109
5110    match timeout_at(deadline, receiver).await {
5111        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5112        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5113        // and those prove it is alive even though the probe failed.
5114        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5115            response.health_report().ok_or_else(|| {
5116                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5117            })
5118        }
5119        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5120            format!("health.check rejected: {}", body.message),
5121        )),
5122        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5123            Err(HealthProbeError::lane_dead(message))
5124        }
5125        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5126            Err(HealthProbeError::bad_answer(message))
5127        }
5128        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5129            Err(HealthProbeError::bad_answer(format!(
5130                "expected module-control op '{expected}', got '{actual}'"
5131            )))
5132        }
5133        // A reply that crosses the deadline before this waiter observes it is
5134        // still proof of life. The forwarding path records its end-to-end latency
5135        // before delivering this classification.
5136        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5137            "module answered health.check after its daemon deadline",
5138        )),
5139        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5140            "health.check waiter was canceled before the module responded",
5141        )),
5142        Err(_) => {
5143            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5144            Err(HealthProbeError::no_answer(format!(
5145                "module did not answer health.check within {probe_budget:?}"
5146            )))
5147        }
5148    }
5149}
5150
5151#[allow(clippy::too_many_arguments)]
5152async fn handle_health_report(
5153    spec: &ModuleSpec,
5154    runtime: &SupervisorRuntimeConfig,
5155    registry: &Registry,
5156    process_liveness: &SupervisorProcessLiveness,
5157    snapshot: &SharedSnapshot,
5158    child: &mut Option<SupervisedChild>,
5159    report: HealthReport,
5160    now_ms: u64,
5161) {
5162    let status = supervisor_health_status(report.status);
5163    let detail = report.detail.clone();
5164    let metrics = truncate_health_metrics(report.metrics);
5165    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5166        state.health.status = status;
5167        state.health.last_probe_ms = Some(now_ms);
5168        state.health.detail = detail.clone();
5169        state.health.metrics = metrics.clone();
5170        state.health.consecutive_failures = 0;
5171    });
5172
5173    let action = match report.status {
5174        HealthStatus::Ok => return,
5175        HealthStatus::Degraded => runtime.health.on_degraded,
5176        HealthStatus::Failing => runtime.health.on_failing,
5177    };
5178    apply_l3_health_action(
5179        spec,
5180        runtime,
5181        registry,
5182        process_liveness,
5183        snapshot,
5184        child,
5185        status,
5186        detail.as_deref(),
5187        action,
5188        now_ms,
5189    )
5190    .await;
5191}
5192
5193#[allow(clippy::too_many_arguments)]
5194async fn handle_health_probe_failure(
5195    spec: &ModuleSpec,
5196    runtime: &SupervisorRuntimeConfig,
5197    registry: &Registry,
5198    process_liveness: &SupervisorProcessLiveness,
5199    snapshot: &SharedSnapshot,
5200    child: &mut Option<SupervisedChild>,
5201    err: HealthProbeError,
5202    now_ms: u64,
5203) {
5204    let threshold = runtime.health.failure_threshold.max(1);
5205    let mut failures = 0;
5206    // Carry the evidence class into the operator-visible detail. Without it,
5207    // "module did not answer within 5s" and "the control lane is gone" are two
5208    // prose strings in the same field, and the reader has to know the codebase to
5209    // tell which one is proof of anything.
5210    let detail = format!("[{}] {err}", err.label());
5211    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5212        // A failed wire probe invalidates the last report, even before the
5213        // restart threshold. HTTP probes already mark failures as Failing.
5214        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5215            state.health.status = SupervisorHealthStatus::Unknown;
5216        }
5217        state.health.last_probe_ms = Some(now_ms);
5218        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5219        state.health.detail = Some(detail.clone());
5220        state.health.metrics = None;
5221        failures = state.health.consecutive_failures;
5222    });
5223
5224    if failures < threshold {
5225        warn!(
5226            module_id = %spec.module_id,
5227            consecutive_failures = failures,
5228            threshold,
5229            evidence = err.label(),
5230            detail = %detail,
5231            "health.check probe failed"
5232        );
5233        return;
5234    }
5235
5236    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5237        state.state = ModuleState::Unresponsive;
5238        state.health.status = SupervisorHealthStatus::Unresponsive;
5239    });
5240    // The evidence class is logged at the kill site because this is the line an
5241    // operator reads after an unexplained restart. A streak of `no-answer` under
5242    // machine load is the known false-positive shape; a `lane-dead` is not.
5243    if runtime.health.critical {
5244        error!(
5245            module_id = %spec.module_id,
5246            status = "unresponsive",
5247            evidence = err.label(),
5248            detail = %detail,
5249            "critical module health alert"
5250        );
5251    } else {
5252        warn!(
5253            module_id = %spec.module_id,
5254            status = "unresponsive",
5255            evidence = err.label(),
5256            detail = %detail,
5257            "module health threshold breached"
5258        );
5259    }
5260    if let Err(err) = health_restart_child(
5261        spec,
5262        runtime,
5263        registry,
5264        process_liveness,
5265        snapshot,
5266        child,
5267        SupervisorHealthStatus::Unresponsive,
5268        Some(&detail),
5269        now_ms,
5270    )
5271    .await
5272    {
5273        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5274    }
5275}
5276
5277#[allow(clippy::too_many_arguments)]
5278async fn apply_l3_health_action(
5279    spec: &ModuleSpec,
5280    runtime: &SupervisorRuntimeConfig,
5281    registry: &Registry,
5282    process_liveness: &SupervisorProcessLiveness,
5283    snapshot: &SharedSnapshot,
5284    child: &mut Option<SupervisedChild>,
5285    status: SupervisorHealthStatus,
5286    detail: Option<&str>,
5287    action: HealthAction,
5288    now_ms: u64,
5289) {
5290    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5291    match action {
5292        HealthAction::Report => {
5293            info!(
5294                module_id = %spec.module_id,
5295                status = ?status,
5296                detail,
5297                "module reported non-ok health"
5298            );
5299        }
5300        HealthAction::Alert => {
5301            error!(
5302                module_id = %spec.module_id,
5303                status = ?status,
5304                detail,
5305                "module health alert"
5306            );
5307        }
5308        HealthAction::Restart => {
5309            if let Err(err) = health_restart_child(
5310                spec,
5311                runtime,
5312                registry,
5313                process_liveness,
5314                snapshot,
5315                child,
5316                status,
5317                detail,
5318                now_ms,
5319            )
5320            .await
5321            {
5322                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5323            }
5324        }
5325    }
5326}
5327
5328#[allow(clippy::too_many_arguments)]
5329async fn health_restart_child(
5330    spec: &ModuleSpec,
5331    runtime: &SupervisorRuntimeConfig,
5332    registry: &Registry,
5333    process_liveness: &SupervisorProcessLiveness,
5334    snapshot: &SharedSnapshot,
5335    child: &mut Option<SupervisedChild>,
5336    status: SupervisorHealthStatus,
5337    detail: Option<&str>,
5338    now_ms: u64,
5339) -> Result<(), SuperviseError> {
5340    let (enabled, schedule) = {
5341        let mut state = lock_snapshot(snapshot)?;
5342        let enabled = state.enabled;
5343        let schedule = if enabled {
5344            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5345        } else {
5346            None
5347        };
5348        (enabled, schedule)
5349    };
5350
5351    if !enabled {
5352        return Err(SuperviseError::Disabled {
5353            module_id: spec.module_id.clone(),
5354        });
5355    }
5356
5357    if schedule.is_none() {
5358        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5359        error!(
5360            module_id = %spec.module_id,
5361            status = ?status,
5362            detail,
5363            max_restarts = runtime.restart_policy.max_restarts,
5364            window_secs = runtime.restart_policy.window.as_secs(),
5365            reason = %runtime.restart_policy.budget_exhausted_detail(),
5366            "health restart budget exhausted; marking module failed"
5367        );
5368        let stop_notice = begin_forwarding_drain_if_configured(
5369            spec,
5370            runtime,
5371            registry,
5372            snapshot,
5373            Some(true),
5374            RouteCloseReason::Disable,
5375        )
5376        .await?;
5377        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5378            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5379        })?;
5380        drain_optional_child(
5381            &spec.module_id,
5382            spec.protocol,
5383            stop_notice,
5384            registry,
5385            runtime.forwarding.as_deref(),
5386            snapshot,
5387            &runtime.terminal_ring,
5388            &runtime.spawn_events,
5389            child,
5390            runtime.drain_timeout,
5391            ModuleState::Failed,
5392            Some(true),
5393        )
5394        .await?;
5395        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5396        return Ok(());
5397    }
5398
5399    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5400    let mut restart_count = 0;
5401    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5402        restart_count = state.crash_restarts.len();
5403        state.state = ModuleState::Unresponsive;
5404        state.health.status = status;
5405        state.health.last_action = Some(HealthAction::Restart.to_string());
5406        state.health.last_action_ms = Some(now_ms);
5407    })?;
5408    warn!(
5409        module_id = %spec.module_id,
5410        status = ?status,
5411        detail,
5412        restart_count,
5413        restart_in_window = schedule.restart_in_window,
5414        delay_ms = schedule.delay.as_millis() as u64,
5415        "health-triggered module restart"
5416    );
5417
5418    let stop_notice = begin_forwarding_drain_if_configured(
5419        spec,
5420        runtime,
5421        registry,
5422        snapshot,
5423        Some(true),
5424        RouteCloseReason::Restart,
5425    )
5426    .await?;
5427    drain_optional_child(
5428        &spec.module_id,
5429        spec.protocol,
5430        stop_notice,
5431        registry,
5432        runtime.forwarding.as_deref(),
5433        snapshot,
5434        &runtime.terminal_ring,
5435        &runtime.spawn_events,
5436        child,
5437        runtime.drain_timeout,
5438        ModuleState::Restarting,
5439        Some(true),
5440    )
5441    .await?;
5442    schedule_respawn(
5443        runtime,
5444        snapshot,
5445        &spec.module_id,
5446        schedule.delay,
5447        RespawnKind::Spawn,
5448    )
5449}
5450
5451fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5452    if let Some(reply) = runtime
5453        .deferred_reload_reply
5454        .lock()
5455        .unwrap_or_else(|p| p.into_inner())
5456        .take()
5457    {
5458        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5459            module_id: module_id.to_string(),
5460            reason: reason.to_string(),
5461        }));
5462    }
5463}
5464
5465fn schedule_respawn(
5466    runtime: &SupervisorRuntimeConfig,
5467    snapshot: &SharedSnapshot,
5468    module_id: &str,
5469    delay: Duration,
5470    kind: RespawnKind,
5471) -> Result<(), SuperviseError> {
5472    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5473    update_snapshot(snapshot, Some(module_id), |state| {
5474        state.respawn_pending = true
5475    })?;
5476    *runtime
5477        .scheduled_respawn
5478        .lock()
5479        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5480        deadline: Instant::now() + delay,
5481        kind,
5482    });
5483    Ok(())
5484}
5485
5486fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5487    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5488        state.health.last_action = Some(action);
5489        state.health.last_action_ms = Some(now_ms);
5490    });
5491}
5492
5493fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5494    match status {
5495        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5496        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5497        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5498    }
5499}
5500
5501/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5502/// returned to every `supervisor.list` and `supervisor.health` caller.
5503///
5504/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5505/// path: that request exists to return a module's complete metrics object, and
5506/// `ck health <module-id>` documents it as the way to see what the cached view
5507/// truncates. The asymmetry is the feature.
5508///
5509/// So a new caller must decide which side it is on rather than assume the cap is
5510/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5511/// the truncation that path exists to avoid.
5512fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5513    let metrics = metrics?;
5514    match serde_json::to_vec(&metrics) {
5515        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5516            "truncated": true,
5517            "original_bytes": encoded.len(),
5518        })),
5519        Ok(_) | Err(_) => Some(metrics),
5520    }
5521}
5522
5523/// Spread health probes so a fleet-wide restart does not converge them.
5524///
5525/// The delay is derived from the module id and probe index rather than a random
5526/// source, so it is deterministic per module: a module keeps its own offset
5527/// across daemon restarts instead of re-rolling into a collision.
5528fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5529    if cadence.is_zero() {
5530        return Duration::ZERO;
5531    }
5532    let cadence_ms = cadence.as_millis() as u64;
5533    // This early return is REDUNDANT, deliberately, and a mutation run will show
5534    // it surviving removal. Recording why here so the next person to notice does
5535    // not have to re-derive it:
5536    //
5537    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5538    //   a zero cadence and builds the Duration from whole milliseconds, so a
5539    //   sub-millisecond cadence cannot come from config.
5540    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5541    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5542    //   -- exactly what this returns.
5543    //
5544    // Kept as a guard against a future widening of the config parser (accepting
5545    // microseconds, say), which would make the sub-millisecond case reachable.
5546    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5547    // divides by zero. Remove this and nothing changes.
5548    if cadence_ms == 0 {
5549        return cadence;
5550    }
5551    // Note that this never returns less than one cadence, including for the FIRST
5552    // probe. So a freshly registered module reports health `unknown` for a full
5553    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5554    // ready to answer.
5555    //
5556    // That is a property of the supervisor's schedule, not of any module: an
5557    // operator watching a restart sees `unknown` and cannot tell it from a module
5558    // that is slow to warm. Measured on two unrelated modules, both flipping to
5559    // `ok` between 22s and 32s after restart.
5560    //
5561    // Left as-is because spreading the first probe is what keeps a fleet-wide
5562    // restart from firing fourteen simultaneous probes into a cold machine. The
5563    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5564    // that thundering herd for a faster first reading.
5565    let jitter_span = (cadence_ms / 10).max(1);
5566    let hash = module_id.as_bytes().iter().fold(
5567        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5568        |acc, byte| {
5569            acc.wrapping_mul(1099511628211)
5570                .wrapping_add(u64::from(*byte))
5571        },
5572    );
5573    cadence + Duration::from_millis(hash % jitter_span)
5574}
5575
5576#[cfg(test)]
5577mod tests {
5578    use super::*;
5579
5580    #[test]
5581    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5582        let handle = SupervisorHandle::new();
5583        let module_id = "readded-tombstone";
5584        handle.record_rescan_removal(module_id);
5585        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5586
5587        handle.apply_identity_configuration(&ModuleSpec {
5588            module_id: module_id.to_string(),
5589            program: PathBuf::from("/test/module"),
5590            args: Vec::new(),
5591            env: Vec::new(),
5592            reserved: false,
5593            reserved_prefixes: Vec::new(),
5594            protocol: ModuleProtocol::Subc,
5595            overlap: Default::default(),
5596        });
5597
5598        assert!(
5599            handle.removal_tombstone_age_ms(module_id).is_none(),
5600            "a re-added module must not retain a stale removal tombstone"
5601        );
5602    }
5603
5604    /// What one module's owner looked like from the control plane at the
5605    /// instant after its first process was spawned.
5606    #[derive(Debug, PartialEq, Eq)]
5607    struct OwnerInSpawnWindow {
5608        module_id: String,
5609        configured: bool,
5610        on_roster: bool,
5611        admission_refusal: Option<&'static str>,
5612    }
5613
5614    /// A supervised module's process can connect, register, sync its scopes
5615    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5616    /// the module on the roster until afterward; configured supervision does
5617    /// so before spawning. In both cases the owner must
5618    /// already read as configured, so a scoped `route.open` against it is
5619    /// refused as retryable `scope_not_synced` and not as terminal
5620    /// `scope_not_live` ("will never sync").
5621    ///
5622    /// The hook runs just after each initial spawn attempt, before returning to
5623    /// the caller, so no race with a real child's registration is needed.
5624    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5625    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5626        use crate::scopes::ScopeTable;
5627        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5628
5629        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5630        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5631            module_id: module_id.to_string(),
5632            program,
5633            args: Vec::new(),
5634            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5635                .into_iter()
5636                .map(|key| (key.to_string(), dir.path().display().to_string()))
5637                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5638                .collect(),
5639            reserved: false,
5640            reserved_prefixes: Vec::new(),
5641            protocol: ModuleProtocol::Subc,
5642            overlap: Default::default(),
5643        };
5644        let live = super::terminal_history_tests::fake_aft_stub_path();
5645        let missing = dir.path().join("definitely-missing-module");
5646
5647        let handle = SupervisorHandle::new();
5648        let mut supervisor =
5649            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5650                .with_handle(handle.clone());
5651        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5652        let hook_handle = handle.clone();
5653        let hook_observed = Arc::clone(&observed);
5654        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5655            // Exactly what the control plane computes for a scoped route.open
5656            // naming this module as the owner of a scope it has not synced.
5657            let configured = hook_handle.is_configured(module_id);
5658            let selector = ScopeSelector::new(
5659                Principal::Reserved {
5660                    module_id: module_id.to_string(),
5661                },
5662                "s".to_string(),
5663                Some(1),
5664            );
5665            let carrier = Principal::Reserved {
5666                module_id: "carrier".to_string(),
5667            };
5668            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5669                .admit(&carrier, module_id, &selector, configured)
5670            {
5671                Ok(_) => None,
5672                Err(refusal) => Some(refusal.code),
5673            };
5674            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5675                module_id: module_id.to_string(),
5676                configured,
5677                on_roster: hook_handle.get(module_id).is_some(),
5678                admission_refusal,
5679            });
5680        })));
5681
5682        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5683        let configured = supervisor
5684            .supervise_configured(stub("configured", live.clone()), true)
5685            .unwrap();
5686        let with_health = supervisor
5687            .supervise_configured_with_health(
5688                stub("with-health", live.clone()),
5689                true,
5690                HealthConfig::default(),
5691                None,
5692                RestartPolicy::default(),
5693            )
5694            .unwrap();
5695        // The failed-spawn path still puts the module on the roster (as
5696        // failed), so it is configured throughout.
5697        let failed = supervisor
5698            .supervise_configured_with_health(
5699                stub("failed-spawn", missing.clone()),
5700                true,
5701                HealthConfig::default(),
5702                None,
5703                RestartPolicy::default(),
5704            )
5705            .unwrap();
5706        // A failed plain `spawn` puts nothing on the roster, so its mark is
5707        // taken back once the spawn has failed.
5708        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5709
5710        let expected = [
5711            "plain",
5712            "configured",
5713            "with-health",
5714            "failed-spawn",
5715            "spawn-error",
5716        ]
5717        .into_iter()
5718        .map(|module_id| OwnerInSpawnWindow {
5719            module_id: module_id.to_string(),
5720            configured: true,
5721            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5722            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5723        })
5724        .collect::<Vec<_>>();
5725        assert_eq!(*observed.lock().unwrap(), expected);
5726
5727        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5728            assert!(
5729                handle.get(module_id).is_some(),
5730                "{module_id} is on the roster"
5731            );
5732            assert!(
5733                handle.is_configured(module_id),
5734                "{module_id} stays configured"
5735            );
5736        }
5737        assert!(handle.get("spawn-error").is_none());
5738        assert!(
5739            !handle.is_configured("spawn-error"),
5740            "a plain spawn that failed must not leave its module marked configured"
5741        );
5742
5743        // Leaving the roster clears the mark with it.
5744        handle.retire("failed-spawn");
5745        assert!(!handle.is_configured("failed-spawn"));
5746
5747        for module in [plain, configured, with_health] {
5748            module.stop().await.unwrap();
5749        }
5750        drop(failed);
5751    }
5752
5753    /// A configured module reads as starting until the supervisor has admitted
5754    /// its first process (exec confirmed, identity durably recorded), even
5755    /// though that process is already running by then and can register over
5756    /// HELLO, which is what lists it in catalog.list. So a client that needs
5757    /// the supervisor state must wait for that state; seeing the module in
5758    /// the catalog does not mean supervisor.list already says running.
5759    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5760    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5761        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5762        let spec = ModuleSpec {
5763            module_id: "spawned-not-admitted".to_string(),
5764            program: super::terminal_history_tests::fake_aft_stub_path(),
5765            args: Vec::new(),
5766            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5767                .into_iter()
5768                .map(|key| (key.to_string(), dir.path().display().to_string()))
5769                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5770                .collect(),
5771            reserved: false,
5772            reserved_prefixes: Vec::new(),
5773            protocol: ModuleProtocol::Subc,
5774            overlap: Default::default(),
5775        };
5776        let handle = SupervisorHandle::new();
5777        let mut supervisor =
5778            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5779                .with_handle(handle.clone());
5780        let observed = Arc::new(Mutex::new(None));
5781        let hook_handle = handle.clone();
5782        let hook_observed = Arc::clone(&observed);
5783        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5784            let state = hook_handle
5785                .get(module_id)
5786                .expect("a configured module is on the roster before it is spawned")
5787                .state()
5788                .unwrap();
5789            *hook_observed.lock().unwrap() = Some(state);
5790        })));
5791
5792        let module = supervisor.supervise_configured(spec, true).unwrap();
5793        assert_eq!(
5794            *observed.lock().unwrap(),
5795            Some(ModuleState::Starting),
5796            "between the spawn and its admission the supervisor still reports starting"
5797        );
5798        assert_eq!(module.state().unwrap(), ModuleState::Running);
5799        module.stop().await.unwrap();
5800    }
5801
5802    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5803        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5804        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5805            snapshot.process_alive = true;
5806            snapshot.pid = Some(41);
5807            snapshot.spawned_at_ms = Some(42);
5808            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5809            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5810            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5811                device: 43,
5812                inode: 44,
5813                #[cfg(windows)]
5814                windows_image: None,
5815            });
5816        })
5817        .unwrap();
5818        snapshot
5819    }
5820
5821    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5822        let snapshot = lock_snapshot(snapshot).unwrap();
5823        assert!(!snapshot.process_alive);
5824        assert_eq!(snapshot.pid, None);
5825        assert_eq!(snapshot.spawned_at_ms, None);
5826        assert_eq!(snapshot.spawned_from, None);
5827        assert_eq!(snapshot.launch_spec_digest, None);
5828        assert_eq!(snapshot.spawned_file_identity, None);
5829    }
5830
5831    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5832    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5833        let supervisor =
5834            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5835        let mut runtime = supervisor.runtime_config();
5836        runtime.test_seed_stale_facts_before_enable_spawn = true;
5837        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5838        let mut child = None;
5839        let spec = ModuleSpec {
5840            module_id: "failed-enable-clears-facts".to_string(),
5841            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5842            args: Vec::new(),
5843            env: Vec::new(),
5844            reserved: false,
5845            reserved_prefixes: Vec::new(),
5846            protocol: ModuleProtocol::Subc,
5847            overlap: Default::default(),
5848        };
5849
5850        let result = set_child_enabled(
5851            &spec,
5852            &runtime,
5853            &supervisor.registry,
5854            &supervisor.process_liveness,
5855            &snapshot,
5856            &mut child,
5857            true,
5858        )
5859        .await;
5860
5861        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5862        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5863        assert_snapshot_process_facts_cleared(&snapshot);
5864    }
5865
5866    #[tokio::test]
5867    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5868        let supervisor =
5869            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5870        let runtime = supervisor.runtime_config();
5871        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5872            ModuleState::Restarting,
5873            true,
5874        )));
5875        let spec = ModuleSpec {
5876            module_id: "start-stranded-restarting".to_string(),
5877            program: super::terminal_history_tests::fake_aft_stub_path(),
5878            args: Vec::new(),
5879            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5880            reserved: false,
5881            reserved_prefixes: Vec::new(),
5882            protocol: ModuleProtocol::None,
5883            overlap: Default::default(),
5884        };
5885        let mut child = None;
5886        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5887        assert!(!super::set_child_enabled(
5888            &spec,
5889            &runtime,
5890            &Registry::default(),
5891            &supervisor.process_liveness,
5892            &snapshot,
5893            &mut child,
5894            true
5895        )
5896        .await
5897        .unwrap());
5898        assert!(child.is_none());
5899        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5900        assert!(super::set_child_enabled(
5901            &spec,
5902            &runtime,
5903            &Registry::default(),
5904            &supervisor.process_liveness,
5905            &snapshot,
5906            &mut child,
5907            true
5908        )
5909        .await
5910        .unwrap());
5911        assert_eq!(
5912            lock_snapshot(&snapshot).unwrap().state,
5913            ModuleState::Running
5914        );
5915        let mut child = child.unwrap();
5916        child.start_kill().unwrap();
5917        child.wait().await.unwrap();
5918    }
5919
5920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5921    async fn failed_reload_spawn_clears_current_process_facts() {
5922        let supervisor =
5923            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5924        let mut runtime = supervisor.runtime_config();
5925        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5926        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5927        let mut child = None;
5928        let spec = ModuleSpec {
5929            module_id: "failed-reload-clears-facts".to_string(),
5930            program: PathBuf::from("/unused/failed-reload-module"),
5931            args: Vec::new(),
5932            env: Vec::new(),
5933            reserved: false,
5934            reserved_prefixes: Vec::new(),
5935            protocol: ModuleProtocol::Subc,
5936            overlap: Default::default(),
5937        };
5938
5939        let result = handle_reload_spawn_failure(
5940            &spec,
5941            &runtime,
5942            &supervisor.process_liveness,
5943            &snapshot,
5944            &mut child,
5945            "forced reload spawn failure".to_string(),
5946        )
5947        .await;
5948
5949        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5950        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5951        assert_snapshot_process_facts_cleared(&snapshot);
5952    }
5953
5954    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5955    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5956        let supervisor =
5957            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5958        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5959        let module = supervisor.supervised_module(
5960            ModuleSpec {
5961                module_id: "drop-clears-facts".to_string(),
5962                program: PathBuf::from("/unused/drop-module"),
5963                args: Vec::new(),
5964                env: Vec::new(),
5965                reserved: false,
5966                reserved_prefixes: Vec::new(),
5967                protocol: ModuleProtocol::Subc,
5968                overlap: Default::default(),
5969            },
5970            supervisor.runtime_config(),
5971            Arc::clone(&snapshot),
5972            None,
5973        );
5974        assert!(!module
5975            .inner
5976            .monitor
5977            .lock()
5978            .unwrap()
5979            .as_ref()
5980            .unwrap()
5981            .is_finished());
5982
5983        drop(module);
5984
5985        assert_eq!(
5986            lock_snapshot(&snapshot).unwrap().state,
5987            ModuleState::Stopped
5988        );
5989        assert_snapshot_process_facts_cleared(&snapshot);
5990    }
5991
5992    #[cfg(unix)]
5993    #[tokio::test]
5994    async fn rescan_preserves_running_protocol_until_respawn() {
5995        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5996        let initial = ModuleSpec {
5997            module_id: "rescan-protocol".into(),
5998            program: PathBuf::from("/bin/sleep"),
5999            args: vec!["60".into()],
6000            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6001                .into_iter()
6002                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6003                .collect(),
6004            reserved: false,
6005            reserved_prefixes: vec![],
6006            protocol: ModuleProtocol::None,
6007            overlap: Default::default(),
6008        };
6009        let supervisor =
6010            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6011        let module = supervisor.spawn(initial.clone()).unwrap();
6012        assert!(module.status().unwrap().live);
6013        let mut next = initial;
6014        next.protocol = ModuleProtocol::Subc;
6015        module
6016            .update_configuration(next.clone(), HealthConfig::default(), None)
6017            .await
6018            .unwrap();
6019        assert!(
6020            module.status().unwrap().live,
6021            "rescan must not require HELLO from the old non-wire process"
6022        );
6023        let runtime = supervisor.runtime_config();
6024        let action = on_child_exit(
6025            &next,
6026            RestartPolicy::default(),
6027            &supervisor.registry,
6028            &module.inner.snapshot,
6029            &runtime.terminal_ring,
6030            &runtime.spawn_events,
6031            &runtime.child_roster,
6032            ExitReport {
6033                kind: ExitKind::Clean,
6034                code: Some(0),
6035                signal: None,
6036                at_ms: unix_ms_now(),
6037            },
6038        )
6039        .await;
6040        assert!(
6041            matches!(action, NextAction::Restart { .. }),
6042            "the old non-wire process's clean exit must restart"
6043        );
6044        module.drain().await.unwrap();
6045    }
6046
6047    #[cfg(unix)]
6048    #[tokio::test]
6049    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6050        use std::os::unix::fs::PermissionsExt;
6051        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6052        let script = dir.join("module.sh");
6053        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6054        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6055        let record_path = dir.join("live-children.json");
6056        let supervisor =
6057            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6058                .with_live_children_record(&record_path);
6059        for (program, args) in [
6060            (PathBuf::from("sleep"), vec!["60".into()]),
6061            (script, vec![]),
6062        ] {
6063            let spec = ModuleSpec {
6064                module_id: "image-identity".into(),
6065                program: program.clone(),
6066                args,
6067                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6068                    .into_iter()
6069                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6070                    .collect(),
6071                reserved: false,
6072                reserved_prefixes: vec![],
6073                protocol: ModuleProtocol::None,
6074                overlap: Default::default(),
6075            };
6076            let module = supervisor.spawn(spec).unwrap();
6077            #[cfg(target_os = "macos")]
6078            {
6079                // SETEXEC confirmation is asynchronous; the orphan record must
6080                // identify the final image, never the intermediate trampoline.
6081                let deadline = Instant::now() + Duration::from_secs(5);
6082                while crate::live_children::read_record(&record_path)
6083                    .unwrap()
6084                    .iter()
6085                    .all(|entry| entry.executable.is_none())
6086                {
6087                    assert!(Instant::now() < deadline, "module image was not confirmed");
6088                    tokio::time::sleep(Duration::from_millis(5)).await;
6089                }
6090            }
6091            let entry = crate::live_children::read_record(&record_path)
6092                .unwrap()
6093                .pop()
6094                .unwrap();
6095            let observed = subc_os::Process::open(entry.pid)
6096                .unwrap()
6097                .unwrap()
6098                .observe()
6099                .unwrap();
6100            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6101            module.drain().await.unwrap();
6102            assert_eq!(
6103                verdict,
6104                crate::live_children::IdentityVerdict::Matches,
6105                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6106            );
6107        }
6108    }
6109
6110    #[cfg(unix)]
6111    fn http_fixture(
6112        dir: &std::path::Path,
6113        url: &str,
6114        threshold: u32,
6115    ) -> crate::daemon_config::ConfiguredModule {
6116        let path = dir.join("subc.jsonc");
6117        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6118            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6119            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6120            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6121            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6122        }}}).to_string()).unwrap();
6123        crate::daemon_config::load(&path)
6124            .unwrap()
6125            .unwrap()
6126            .modules
6127            .pop()
6128            .unwrap()
6129    }
6130
6131    #[cfg(unix)]
6132    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6133        timeout(Duration::from_secs(5), async {
6134            loop {
6135                if module.status().unwrap().health.status == status {
6136                    break;
6137                }
6138                sleep(Duration::from_millis(5)).await;
6139            }
6140        })
6141        .await
6142        .unwrap_or_else(|_| {
6143            panic!(
6144                "expected {status:?}, got {:?}",
6145                module.status().unwrap().health
6146            )
6147        });
6148    }
6149
6150    #[cfg(unix)]
6151    #[tokio::test]
6152    async fn http_health_status_flips_ok_failing_ok() {
6153        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6154        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6155        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6156        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6157        let serving_status = status.clone();
6158        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6159        let server = tokio::spawn(async move {
6160            loop {
6161                let (mut stream, _) = listener.accept().await.unwrap();
6162                let mut request = [0u8; 2048];
6163                let count = stream.read(&mut request).await.unwrap();
6164                assert!(count > 0, "a probe must send an HTTP request");
6165                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6166                let body = if code == 200 {
6167                    "ready"
6168                } else {
6169                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6170                };
6171                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6172                let _ = stream.write_all(response.as_bytes()).await;
6173            }
6174        });
6175        let configured = http_fixture(&dir, &url, 1000);
6176        let module =
6177            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6178                .supervise_configured_with_health(
6179                    configured.module_spec(),
6180                    true,
6181                    configured.health,
6182                    configured.drain_timeout_ms,
6183                    configured.restart,
6184                )
6185                .unwrap();
6186        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6187        status.store(503, std::sync::atomic::Ordering::SeqCst);
6188        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6189        assert!(module
6190            .status()
6191            .unwrap()
6192            .health
6193            .detail
6194            .unwrap()
6195            .contains("scratch failure"));
6196        status.store(200, std::sync::atomic::Ordering::SeqCst);
6197        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6198        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6199        let before = module.status().unwrap();
6200        let (spec, mut health) = module.configuration().unwrap();
6201        health.http = None;
6202        module
6203            .update_configuration(spec.clone(), health.clone(), Some(10))
6204            .await
6205            .unwrap();
6206        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6207        health.http = Some(url);
6208        module
6209            .update_configuration(spec, health, Some(10))
6210            .await
6211            .unwrap();
6212        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6213        assert_eq!(
6214            module.status().unwrap().pid,
6215            before.pid,
6216            "changing a probe must apply live, not restart its process"
6217        );
6218        let (spec, mut health) = module.configuration().unwrap();
6219        health.failure_threshold = 2;
6220        module
6221            .update_configuration(spec, health, Some(10))
6222            .await
6223            .unwrap();
6224        status.store(503, std::sync::atomic::Ordering::SeqCst);
6225        timeout(Duration::from_secs(5), async {
6226            while module.status().unwrap().spawn_generation == before.spawn_generation {
6227                sleep(Duration::from_millis(5)).await;
6228            }
6229        })
6230        .await
6231        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6232        module.drain().await.unwrap();
6233        server.abort();
6234    }
6235
6236    #[cfg(unix)]
6237    #[tokio::test]
6238    async fn http_health_replacement_recovers_after_one_restart() {
6239        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6240        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6241        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6242        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6243        let mut configured = http_fixture(&dir, &url, 2);
6244        configured.restart.max_restarts = 2;
6245        let module =
6246            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6247                .supervise_configured_with_health(
6248                    configured.module_spec(),
6249                    true,
6250                    configured.health,
6251                    configured.drain_timeout_ms,
6252                    configured.restart,
6253                )
6254                .unwrap();
6255        let first_generation = module.status().unwrap().spawn_generation;
6256        let observed = module.clone();
6257        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6258        let server = tokio::spawn(async move {
6259            let mut generation = first_generation;
6260            let mut probes = 0;
6261            loop {
6262                let (mut stream, _) = listener.accept().await.unwrap();
6263                let mut request = [0; 2048];
6264                assert!(stream.read(&mut request).await.unwrap() > 0);
6265                let current = observed.status().unwrap().spawn_generation;
6266                if current != generation {
6267                    generation = current;
6268                    probes = 0;
6269                }
6270                probes += 1;
6271                // The original process never recovers. Each replacement has one
6272                // transient miss, then answers successfully for its remaining life.
6273                let healthy = generation != first_generation && probes > 1;
6274                let code = if healthy { 200 } else { 503 };
6275                let response = format!(
6276                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6277                );
6278                let _ = stream.write_all(response.as_bytes()).await;
6279                if healthy {
6280                    healthy_tx.send(()).await.unwrap();
6281                }
6282            }
6283        });
6284        timeout(Duration::from_secs(5), async {
6285            let mut healthy_probes = 0;
6286            loop {
6287                if module.state().unwrap() == ModuleState::Failed {
6288                    break;
6289                }
6290                while healthy_rx.try_recv().is_ok() {
6291                    healthy_probes += 1;
6292                }
6293                if healthy_probes >= 3
6294                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6295                {
6296                    break;
6297                }
6298                sleep(Duration::from_millis(5)).await;
6299            }
6300        })
6301        .await
6302        .expect("the replacement must settle healthy without exhausting its restart budget");
6303        let status = module.status().unwrap();
6304        module.drain().await.unwrap();
6305        server.abort();
6306        assert_eq!(status.state, ModuleState::Running);
6307        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6308        assert_eq!(status.health.consecutive_failures, 0);
6309        assert_eq!(status.lifetime_restarts, 1);
6310        assert_eq!(status.restart_count, 1);
6311    }
6312
6313    #[cfg(unix)]
6314    #[tokio::test]
6315    async fn http_health_sustained_failure_exhausts_restart_budget() {
6316        use std::collections::BTreeMap;
6317        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6318        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6319        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6320        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6321        let mut configured = http_fixture(&dir, &url, 2);
6322        configured.restart.max_restarts = 2;
6323        let module =
6324            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6325                .supervise_configured_with_health(
6326                    configured.module_spec(),
6327                    true,
6328                    configured.health,
6329                    configured.drain_timeout_ms,
6330                    configured.restart,
6331                )
6332                .unwrap();
6333        let observed = module.clone();
6334        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6335        let served = probes.clone();
6336        let server = tokio::spawn(async move {
6337            loop {
6338                let (mut stream, _) = listener.accept().await.unwrap();
6339                let mut request = [0; 2048];
6340                assert!(stream.read(&mut request).await.unwrap() > 0);
6341                *served
6342                    .lock()
6343                    .unwrap()
6344                    .entry(observed.status().unwrap().spawn_generation)
6345                    .or_default() += 1;
6346                let _ = stream
6347                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6348                    .await;
6349            }
6350        });
6351        timeout(Duration::from_secs(5), async {
6352            while module.state().unwrap() != ModuleState::Failed {
6353                sleep(Duration::from_millis(5)).await;
6354            }
6355        })
6356        .await
6357        .expect("sustained failures must still exhaust the health restart budget");
6358        let status = module.status().unwrap();
6359        server.abort();
6360        assert_eq!(status.restart_count, 2);
6361        assert_eq!(status.lifetime_restarts, 2);
6362        assert!(!status.process_alive);
6363        assert_eq!(
6364            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6365            vec![2, 2, 2]
6366        );
6367    }
6368
6369    #[cfg(unix)]
6370    #[tokio::test]
6371    async fn spawn_clears_the_previous_process_health() {
6372        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6373        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6374        configured.health.cadence = Duration::from_secs(3600);
6375        let module =
6376            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6377                .supervise_configured_with_health(
6378                    configured.module_spec(),
6379                    false,
6380                    configured.health,
6381                    configured.drain_timeout_ms,
6382                    configured.restart,
6383                )
6384                .unwrap();
6385        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6386            state.health.status = SupervisorHealthStatus::Unresponsive;
6387            state.health.consecutive_failures = 3;
6388            state.health.detail = Some("old process refused connections".into());
6389            state.health.metrics = Some(serde_json::json!({"old": true}));
6390            state.health.last_probe_ms = Some(123);
6391            state.health.last_action = Some("restart".into());
6392            state.health.last_action_ms = Some(124);
6393        })
6394        .unwrap();
6395        module.set_enabled(true).await.unwrap();
6396        let health = module.status().unwrap().health;
6397        module.drain().await.unwrap();
6398        assert_eq!(health.consecutive_failures, 0);
6399        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6400        assert_eq!(health.detail, None);
6401        assert_eq!(health.metrics, None);
6402        assert_eq!(health.last_probe_ms, None);
6403        assert_eq!(health.last_action.as_deref(), Some("restart"));
6404        assert_eq!(health.last_action_ms, Some(124));
6405    }
6406
6407    #[cfg(unix)]
6408    #[tokio::test]
6409    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6410        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6411        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6412        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6413        drop(unused);
6414        let configured = http_fixture(&dir, &url, 2);
6415        let module =
6416            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6417                .supervise_configured_with_health(
6418                    configured.module_spec(),
6419                    true,
6420                    configured.health,
6421                    configured.drain_timeout_ms,
6422                    configured.restart,
6423                )
6424                .unwrap();
6425        let before = module.status().unwrap().spawn_generation;
6426        timeout(Duration::from_secs(5), async {
6427            loop {
6428                let status = module.status().unwrap();
6429                if status.spawn_generation > before {
6430                    assert!(status.lifetime_restarts > 0);
6431                    break;
6432                }
6433                sleep(Duration::from_millis(5)).await;
6434            }
6435        })
6436        .await
6437        .expect("sustained HTTP refusal must trigger the health restart policy");
6438        module.drain().await.unwrap();
6439    }
6440
6441    #[cfg(unix)]
6442    #[tokio::test]
6443    async fn http_health_timeout_honours_deadline() {
6444        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6445        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6446        let server = tokio::spawn(async move {
6447            let _held = listener.accept().await.unwrap();
6448            std::future::pending::<()>().await;
6449        });
6450        let error = timeout(
6451            Duration::from_secs(1),
6452            probe_http_health(&url, Duration::from_millis(10)),
6453        )
6454        .await
6455        .expect("the probe must enforce its own deadline")
6456        .unwrap_err();
6457        server.abort();
6458        assert!(error.to_string().contains("timed out"));
6459    }
6460
6461    #[cfg(unix)]
6462    #[tokio::test]
6463    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6464        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6465        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6466        let url = format!(
6467            "http://localhost:{}/healthz",
6468            listener.local_addr().unwrap().port()
6469        );
6470        let server = tokio::spawn(async move {
6471            let (mut stream, _) = listener.accept().await.unwrap();
6472            let mut request = [0u8; 2048];
6473            assert!(stream.read(&mut request).await.unwrap() > 0);
6474            stream
6475                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6476                .await
6477                .unwrap();
6478        });
6479        // The deadline only bounds a hang. A probe that never tried the IPv6
6480        // address would be refused on 127.0.0.1 and fail at once, so a longer
6481        // deadline does not weaken the assertion; one second timed out under a
6482        // loaded parallel test run.
6483        assert_eq!(
6484            probe_http_health(&url, Duration::from_secs(10))
6485                .await
6486                .unwrap()
6487                .status,
6488            HealthStatus::Ok
6489        );
6490        server.await.unwrap();
6491    }
6492
6493    #[cfg(unix)]
6494    #[tokio::test]
6495    async fn http_health_timeout_keeps_partial_status_and_body() {
6496        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6497        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6498        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6499        let server = tokio::spawn(async move {
6500            let (mut stream, _) = listener.accept().await.unwrap();
6501            let mut request = [0u8; 2048];
6502            assert!(stream.read(&mut request).await.unwrap() > 0);
6503            stream
6504                .write_all(
6505                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6506                )
6507                .await
6508                .unwrap();
6509            std::future::pending::<()>().await;
6510        });
6511        let error = probe_http_health(&url, Duration::from_secs(1))
6512            .await
6513            .unwrap_err()
6514            .to_string();
6515        server.abort();
6516        assert!(
6517            error.contains("timed out")
6518                && error.contains("503 Unavailable")
6519                && error.contains("partial diagnostic"),
6520            "{error}"
6521        );
6522    }
6523
6524    #[cfg(unix)]
6525    #[tokio::test]
6526    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6527        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6528        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6529        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6530        let server = tokio::spawn(async move {
6531            let (mut stream, _) = listener.accept().await.unwrap();
6532            let mut request = [0u8; 2048];
6533            assert!(stream.read(&mut request).await.unwrap() > 0);
6534            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6535            let response = format!(
6536                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6537                body.len()
6538            );
6539            stream.write_all(response.as_bytes()).await.unwrap();
6540        });
6541        let error = probe_http_health(&url, Duration::from_secs(1))
6542            .await
6543            .unwrap_err()
6544            .to_string();
6545        server.await.unwrap();
6546        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6547        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6548        assert!(!error.contains("not-in-diagnostic"));
6549    }
6550
6551    #[cfg(unix)]
6552    #[tokio::test]
6553    async fn http_health_real_nats_server_monitoring() {
6554        if std::process::Command::new("nats-server")
6555            .arg("--version")
6556            .env("XDG_DATA_HOME", std::env::temp_dir())
6557            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6558            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6559            .output()
6560            .is_err()
6561        {
6562            eprintln!(
6563                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6564            );
6565            return;
6566        }
6567        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6568        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6569        let port = monitor.local_addr().unwrap().port();
6570        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6571        let client_port = client.local_addr().unwrap().port();
6572        let config = dir.join("server.conf");
6573        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6574        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6575        configured.program = PathBuf::from("nats-server");
6576        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6577        drop(monitor);
6578        drop(client);
6579        let module =
6580            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6581                .supervise_configured_with_health(
6582                    configured.module_spec(),
6583                    true,
6584                    configured.health,
6585                    configured.drain_timeout_ms,
6586                    configured.restart,
6587                )
6588                .unwrap();
6589        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6590        module.drain().await.unwrap();
6591    }
6592
6593    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6594    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6595        let supervisor =
6596            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6597        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6598        let initial = ModuleSpec {
6599            module_id: "rescan-preserves-spawn-facts".to_string(),
6600            program: PathBuf::from("/spawned/module"),
6601            args: Vec::new(),
6602            env: Vec::new(),
6603            reserved: false,
6604            reserved_prefixes: Vec::new(),
6605            protocol: ModuleProtocol::Subc,
6606            overlap: Default::default(),
6607        };
6608        let module = supervisor.supervised_module(
6609            initial.clone(),
6610            supervisor.runtime_config(),
6611            snapshot,
6612            None,
6613        );
6614        let before = module.status().unwrap();
6615        let mut replacement = initial;
6616        replacement.program = PathBuf::from("/rescanned/replacement-module");
6617
6618        module
6619            .update_configuration(replacement, HealthConfig::default(), None)
6620            .await
6621            .unwrap();
6622
6623        let after = module.status().unwrap();
6624        assert_eq!(after.pid, before.pid);
6625        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6626        assert_eq!(after.spawned_from, before.spawned_from);
6627        drop(module);
6628    }
6629}
6630
6631fn unix_ms_now() -> u64 {
6632    SystemTime::now()
6633        .duration_since(UNIX_EPOCH)
6634        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6635        .unwrap_or(0)
6636}
6637
6638async fn supervise_loop(
6639    mut spec: ModuleSpec,
6640    mut runtime: SupervisorRuntimeConfig,
6641    registry: Arc<Registry>,
6642    process_liveness: Arc<SupervisorProcessLiveness>,
6643    snapshot: SharedSnapshot,
6644    mut child: Option<SupervisedChild>,
6645    mut commands: mpsc::Receiver<SupervisorCommand>,
6646) {
6647    let mut health_probe = HealthProbeRuntime::default();
6648    // Registry writes (including embedded callers) notify this module only.
6649    // Subscribe before the first refresh; watch retains changes that arrive
6650    // while commands or probes are running. No polling fallback is needed.
6651    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6652        Ok(changes) => Some(changes),
6653        Err(err) => {
6654            // A poisoned registry cannot accept further writes, so it cannot
6655            // recover via a timer. Keep exit and command handling alive.
6656            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6657            None
6658        }
6659    };
6660    // All restart backoffs run here, including health and operator requests.
6661    // While one is pending the loop serves commands, so disable or drain can
6662    // cancel the replacement without spawning a process just to stop it.
6663    let mut pending_respawn: Option<PendingRespawn> = None;
6664    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6665    // before anything else so a stop that interrupted a swap runs at once.
6666    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6667    let mut previous_instance_recheck = tokio::time::interval_at(
6668        Instant::now() + Duration::from_secs(5),
6669        Duration::from_secs(5),
6670    );
6671    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6672    loop {
6673        #[cfg(test)]
6674        {
6675            lock_snapshot(&snapshot).unwrap().actor_select = None;
6676        }
6677        #[cfg(target_os = "macos")]
6678        if let Some(active) = child.as_mut() {
6679            active.confirm_privacy_exec().await;
6680        }
6681        if let Some(scheduled) = runtime
6682            .scheduled_respawn
6683            .lock()
6684            .unwrap_or_else(|p| p.into_inner())
6685            .take()
6686        {
6687            pending_respawn = Some(scheduled);
6688        }
6689        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6690            pending_respawn = None;
6691            cancel_deferred_reload(
6692                &runtime,
6693                &spec.module_id,
6694                "respawn cancelled by a supervisor command",
6695            );
6696        }
6697        if child.is_none()
6698            && pending_respawn.is_none()
6699            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6700        {
6701            cancel_deferred_reload(
6702                &runtime,
6703                &spec.module_id,
6704                "respawn cancelled before a replacement was spawned",
6705            );
6706            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6707                state.respawn_pending = false;
6708                state.coalesced_restart_pending = false;
6709                if matches!(
6710                    state.state,
6711                    ModuleState::Restarting
6712                        | ModuleState::Starting
6713                        | ModuleState::Draining
6714                        | ModuleState::Unresponsive
6715                ) {
6716                    error!(module_id = %spec.module_id, state = ?state.state,
6717                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6718                    state.state = ModuleState::Failed;
6719                    clear_current_process_facts(state);
6720                }
6721            });
6722        }
6723        if let Some(command) = requeued.pop_front() {
6724            if !handle_supervisor_command(
6725                command,
6726                &mut spec,
6727                &mut runtime,
6728                &registry,
6729                &process_liveness,
6730                &snapshot,
6731                &mut child,
6732                &mut commands,
6733                &mut requeued,
6734            )
6735            .await
6736            {
6737                return;
6738            }
6739            if child.is_some() || !respawn_still_pending(&snapshot) {
6740                pending_respawn = None;
6741                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6742                    state.respawn_pending = false
6743                });
6744            }
6745            continue;
6746        }
6747        if child.is_some() {
6748            #[cfg(test)]
6749            {
6750                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6751            }
6752            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6753            let wake_after = health_probe.wake_after();
6754            let probe_sleep = async move {
6755                match wake_after {
6756                    Some(delay) => sleep(delay).await,
6757                    None => std::future::pending().await,
6758                }
6759            };
6760            tokio::pin!(probe_sleep);
6761            // A `protocol: "none"` child (a plain process that never registers
6762            // over the subc wire, such as nats-server) with no HTTP health
6763            // check has nothing to probe, so it waits only for its exit or a
6764            // supervisor command. A rescan that adds an HTTP check is a command.
6765            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6766            #[cfg(test)]
6767            {
6768                let registration_pending = wire_child
6769                    && registration_changes
6770                        .as_ref()
6771                        .is_some_and(|changes| changes.has_changed().unwrap());
6772                let mut state = lock_snapshot(&snapshot).unwrap();
6773                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6774                    generation: state.spawn_generation,
6775                    turn: state.actor_turns,
6776                    registered_connection: health_probe.registered_connection,
6777                    next_probe_at: health_probe.next_probe_at,
6778                    wake_after,
6779                });
6780            }
6781            let active_child = child.as_mut().expect("child checked above");
6782            tokio::select! {
6783                wait_result = active_child.wait() => {
6784                    #[cfg(test)]
6785                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6786                    // Every arm below that gives up on the CHILD must keep the
6787                    // supervision task itself alive (child = None, loop
6788                    // continues into command-serving mode). Returning here
6789                    // closes the command channel, which makes the module
6790                    // permanently unrestartable in-band: a clean child exit
6791                    // of an enabled module once wedged the fleet this way
6792                    // ('supervisor command channel is closed') and required a
6793                    // full daemon restart to recover.
6794                    let exit_report = match wait_result {
6795                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6796                        Err(err) => {
6797                            active_child.drain_stderr(&spec.module_id).await;
6798                            fail_after_wait_error(
6799                                &spec.module_id,
6800                                &snapshot,
6801                                &runtime.terminal_ring,
6802                                &runtime.spawn_events,
6803                                format!("failed to wait for supervised module: {err}"),
6804                            );
6805                            untrack_if_registration_released(
6806                                &process_liveness,
6807                                &registry,
6808                                &spec.module_id,
6809                                &snapshot,
6810                            );
6811                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6812                            child = None;
6813                            continue;
6814                        }
6815                    };
6816                    active_child.drain_stderr(&spec.module_id).await;
6817
6818                    let next = on_child_exit(
6819                        &spec,
6820                        runtime.restart_policy,
6821                        &registry,
6822                        &snapshot,
6823                        &runtime.terminal_ring,
6824                        &runtime.spawn_events,
6825                        &runtime.child_roster,
6826                        exit_report,
6827                    ).await;
6828                    runtime.start_requester = None;
6829                    runtime.pending_activity_restart = false;
6830                    // The exit is recorded, so a daemon shutdown may stop
6831                    // waiting for this child (see `SupervisedChild::wait`).
6832                    active_child.release_roster();
6833                    match next {
6834                        NextAction::Stop { registration_released } => {
6835                            if registration_released {
6836                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6837                            }
6838                            child = None;
6839                        }
6840                        NextAction::Restart { schedule } => {
6841                            let delay = schedule.map_or(
6842                                runtime.restart_policy.delay_for_restart(0),
6843                                |schedule| schedule.delay,
6844                            );
6845                            if let Some(schedule) = schedule {
6846                                log_crash_respawn(&spec.module_id, schedule);
6847                            }
6848                            // The exited child is fully recorded at this point,
6849                            // so release it and count the backoff down in the
6850                            // command-serving branch below rather than sleeping
6851                            // here: commands cannot be received from inside this
6852                            // select arm, and an operator disable or drain that
6853                            // arrives during the backoff must cancel the pending
6854                            // respawn instead of waiting for it to spawn first.
6855                            child = None;
6856                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6857                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6858                        }
6859                    }
6860                }
6861                command = commands.recv() => {
6862                    #[cfg(test)]
6863                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6864                    let Some(command) = command else {
6865                        return;
6866                    };
6867                    if !handle_supervisor_command(
6868                        command,
6869                        &mut spec,
6870                        &mut runtime,
6871                        &registry,
6872                        &process_liveness,
6873                        &snapshot,
6874                        &mut child,
6875                        &mut commands,
6876                        &mut requeued,
6877                    ).await {
6878                        return;
6879                    }
6880                }
6881                _ = async {
6882                    match registration_changes.as_mut() {
6883                        Some(changes) => { let _ = changes.changed().await; }
6884                        None => std::future::pending().await,
6885                    }
6886                }, if wire_child => {
6887                    #[cfg(test)]
6888                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6889                }
6890                _ = &mut probe_sleep => {
6891                    #[cfg(test)]
6892                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6893                    if health_probe.due() {
6894                        run_health_probe_cycle(
6895                            &spec,
6896                            &runtime,
6897                            &registry,
6898                            &process_liveness,
6899                            &snapshot,
6900                            &mut child,
6901                        ).await;
6902                        if child.is_none() {
6903                            runtime.start_requester = None;
6904                            runtime.pending_activity_restart = false;
6905                        }
6906                        if child.is_some() {
6907                            health_probe.schedule_next(&spec, runtime.health.cadence);
6908                        }
6909                    }
6910                }
6911            }
6912        } else if let Some(pending) = pending_respawn {
6913            tokio::select! {
6914                _ = sleep_until(pending.deadline) => {
6915                    pending_respawn = None;
6916                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6917                    // A command handled below while the backoff elapsed may
6918                    // have stopped the module; never respawn past an operator's
6919                    // disable or drain.
6920                    if !respawn_still_pending(&snapshot) {
6921                        continue;
6922                    }
6923                    // The daemon began shutting down during the backoff: the
6924                    // spawn would be refused anyway, and refusing it here
6925                    // leaves the module stopped instead of reporting a
6926                    // failed restart.
6927                    if runtime.child_roster.is_closed() {
6928                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6929                            state.state = ModuleState::Stopped;
6930                        });
6931                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6932                        continue;
6933                    }
6934                    if let Err(err) = release_dead_registration(
6935                        &registry,
6936                        runtime.forwarding.as_deref(),
6937                        &snapshot,
6938                        &spec.module_id,
6939                    ).await {
6940                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6941                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6942                        continue;
6943                    }
6944
6945                    if matches!(pending.kind, RespawnKind::Reload) {
6946                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6947                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6948                        if result.is_ok() && runtime.pending_activity_restart {
6949                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6950                        }
6951                        runtime.pending_activity_restart = false;
6952                        if let Some(reply) = reply { let _ = reply.send(result); }
6953                        continue;
6954                    }
6955                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6956                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6957                        Ok(next_child) => {
6958                            child = Some(next_child);
6959                            if runtime.pending_activity_restart {
6960                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6961                                runtime.pending_activity_restart = false;
6962                            }
6963                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6964                        }
6965                        Err(err) => {
6966                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6967                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6968                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6969                        }
6970                    }
6971                }
6972                command = commands.recv() => {
6973                    let Some(command) = command else {
6974                        return;
6975                    };
6976                    if !handle_supervisor_command(
6977                        command,
6978                        &mut spec,
6979                        &mut runtime,
6980                        &registry,
6981                        &process_liveness,
6982                        &snapshot,
6983                        &mut child,
6984                        &mut commands,
6985                        &mut requeued,
6986                    ).await {
6987                        return;
6988                    }
6989                    // Reconcile the pending respawn with what the command did:
6990                    // a start may already have spawned a fresh child,
6991                    // while a disable or drain moved the snapshot out of the
6992                    // state the respawn was counting down from.
6993                    if child.is_some() || !respawn_still_pending(&snapshot) {
6994                        pending_respawn = None;
6995                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6996                    }
6997                }
6998            }
6999        } else {
7000            let command = if lock_snapshot(&snapshot)
7001                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7002            {
7003                tokio::select! {
7004                    command = commands.recv() => command,
7005                    _ = previous_instance_recheck.tick() => {
7006                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7007                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7008                            if enabled && !runtime.child_roster.is_closed() {
7009                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7010                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7011                                    Ok(next_child) => child = Some(next_child),
7012                                    Err(error) => {
7013                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
7014                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7015                                    }
7016                                }
7017                            } else {
7018                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7019                                    state.waiting_for_previous_pid = None;
7020                                    state.waiting_for_previous_detail = None;
7021                                });
7022                            }
7023                        }
7024                        continue;
7025                    }
7026                }
7027            } else {
7028                commands.recv().await
7029            };
7030            let Some(command) = command else {
7031                return;
7032            };
7033            if !handle_supervisor_command(
7034                command,
7035                &mut spec,
7036                &mut runtime,
7037                &registry,
7038                &process_liveness,
7039                &snapshot,
7040                &mut child,
7041                &mut commands,
7042                &mut requeued,
7043            )
7044            .await
7045            {
7046                return;
7047            }
7048        }
7049    }
7050}
7051
7052fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7053    info!(
7054        module_id,
7055        restart_in_window = schedule.restart_in_window,
7056        delay_ms = schedule.delay.as_millis() as u64,
7057        "respawning after crash"
7058    );
7059}
7060
7061/// Whether the respawn a backoff was counting down to is still wanted. A
7062/// disable or drain handled while the backoff elapsed moves the snapshot out
7063/// of `Restarting`, and the operator's stop must win over the pending respawn,
7064/// so every sleep-then-spawn path re-validates against the live snapshot
7065/// instead of assuming the state it left behind still holds.
7066fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7067    matches!(
7068        lock_snapshot(snapshot),
7069        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7070    )
7071}
7072
7073enum NextAction {
7074    Stop {
7075        registration_released: bool,
7076    },
7077    Restart {
7078        schedule: Option<CrashRestartSchedule>,
7079    },
7080}
7081
7082#[allow(clippy::too_many_arguments)]
7083async fn handle_supervisor_command(
7084    command: SupervisorCommand,
7085    spec: &mut ModuleSpec,
7086    runtime: &mut SupervisorRuntimeConfig,
7087    registry: &Arc<Registry>,
7088    process_liveness: &SupervisorProcessLiveness,
7089    snapshot: &SharedSnapshot,
7090    child: &mut Option<SupervisedChild>,
7091    commands: &mut mpsc::Receiver<SupervisorCommand>,
7092    requeued: &mut VecDeque<SupervisorCommand>,
7093) -> bool {
7094    let command =
7095        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7096            match command {
7097                SupervisorCommand::Restart { reply, .. }
7098                | SupervisorCommand::Reload { reply, .. }
7099                | SupervisorCommand::Swap { reply, .. } => {
7100                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7101                    return true;
7102                }
7103                SupervisorCommand::SetEnabled {
7104                    enabled: true,
7105                    reply,
7106                    ..
7107                } => {
7108                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7109                    return true;
7110                }
7111                other => other,
7112            }
7113        } else {
7114            command
7115        };
7116    match command {
7117        SupervisorCommand::Drain { requester, reply } => {
7118            // A plain stop runs no forwarding drain, so nothing reaches the
7119            // module over its connection before the wait: ask by signal.
7120            let result = drain_optional_child(
7121                &spec.module_id,
7122                spec.protocol,
7123                StopNotice::NotSent,
7124                registry,
7125                runtime.forwarding.as_deref(),
7126                snapshot,
7127                &runtime.terminal_ring,
7128                &runtime.spawn_events,
7129                child,
7130                runtime.drain_timeout,
7131                ModuleState::Stopped,
7132                None,
7133            )
7134            .await;
7135            let registration_released = result.is_ok();
7136            if registration_released {
7137                runtime.record_activity_transition(
7138                    "subc.module_stopped",
7139                    &spec.module_id,
7140                    requester,
7141                );
7142            }
7143            let _ = reply.send(result);
7144            if registration_released {
7145                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7146            }
7147            false
7148        }
7149        SupervisorCommand::Retire { requester, reply } => {
7150            let result = async {
7151                let stop_notice = begin_forwarding_drain_if_configured(
7152                    spec,
7153                    runtime,
7154                    registry,
7155                    snapshot,
7156                    None,
7157                    RouteCloseReason::Disable,
7158                )
7159                .await?;
7160                drain_optional_child(
7161                    &spec.module_id,
7162                    spec.protocol,
7163                    stop_notice,
7164                    registry,
7165                    runtime.forwarding.as_deref(),
7166                    snapshot,
7167                    &runtime.terminal_ring,
7168                    &runtime.spawn_events,
7169                    child,
7170                    runtime.drain_timeout,
7171                    ModuleState::Stopped,
7172                    None,
7173                )
7174                .await
7175            }
7176            .await;
7177            let registration_released = result.is_ok();
7178            if registration_released {
7179                runtime.record_activity_transition(
7180                    "subc.module_stopped",
7181                    &spec.module_id,
7182                    requester,
7183                );
7184            }
7185            let _ = reply.send(result);
7186            if registration_released {
7187                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7188            }
7189            false
7190        }
7191        SupervisorCommand::Restart {
7192            requester,
7193            drain_timeout_ms,
7194            received_at_generation,
7195            queued_at,
7196            reply,
7197        } => {
7198            // Without this line a restart that waited in the queue (behind a
7199            // health probe cycle or another command) was invisible: the log
7200            // showed only the drain timing out, minutes after the operator's call.
7201            info!(
7202                module_id = %spec.module_id,
7203                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7204                "restart command dequeued"
7205            );
7206            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7207            // caller whose own request lane rides the module being restarted: the
7208            // caller's in-flight request keeps the drain from quiescing, the drain
7209            // keeps the restart from completing, and the completion keeps the reply
7210            // from releasing the caller — so the drain always timed out and cut the
7211            // initiator with a GOODBYE, even on a healthy module. Replying once the
7212            // restart is validated lets a self-lane caller settle, which is exactly
7213            // what makes the drain succeed. Completion is observable via
7214            // supervisor.list / module status; a post-ack failure lands the module
7215            // in a visible terminal state below rather than in a reply nobody can
7216            // receive.
7217            let validation = match lock_snapshot(snapshot) {
7218                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7219                    module_id: spec.module_id.clone(),
7220                }),
7221                Ok(_) => Ok(()),
7222                Err(err) => Err(err),
7223            };
7224            let initiated = validation.is_ok();
7225            let _ = reply.send(validation);
7226            // A restart asks for a fresh process. Commands run one at a time,
7227            // so a restart queued behind another restart (two operator calls
7228            // in quick succession) is dequeued the moment the first one has
7229            // spawned its replacement -- before that process has sent HELLO.
7230            // Running it would drain and kill the process the first restart
7231            // just produced, which is the opposite of what both callers asked
7232            // for. If a process spawned after this request was received is
7233            // still supervised, the request is already satisfied. Not when the
7234            // configuration changed since that spawn: then the newer process
7235            // predates the spec this restart may exist to apply.
7236            // Generation publication can lag a child that is already serving
7237            // while its identity is recorded. Only process creation after request
7238            // receipt satisfies a restart, not publication after receipt.
7239            let satisfied_by_generation = if initiated
7240                && child
7241                    .as_ref()
7242                    .is_some_and(|child| child.spawned_at > queued_at)
7243            {
7244                lock_snapshot(snapshot).ok().and_then(|state| {
7245                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7246                })
7247            } else {
7248                None
7249            };
7250            let satisfied_by_pending = initiated
7251                && child.is_none()
7252                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7253                    // No child exists yet, so this replacement will necessarily
7254                    // be created after the queued request, unlike a published
7255                    // generation whose process may already have been serving.
7256                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7257                    if pending {
7258                        state.coalesced_restart_pending = true;
7259                    }
7260                    pending
7261                });
7262            if satisfied_by_pending {
7263                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7264            } else if let Some(generation) = satisfied_by_generation {
7265                info!(
7266                    module_id = %spec.module_id,
7267                    received_at_generation,
7268                    "restart already satisfied by generation {generation}; not restarting again"
7269                );
7270            } else if initiated {
7271                // Precedence: this restart's operator override, else the module's
7272                // configured budget (already resolved into the runtime).
7273                let drain_timeout = drain_timeout_ms
7274                    .map(Duration::from_millis)
7275                    .unwrap_or(runtime.drain_timeout);
7276                if let Err(err) = restart_child(
7277                    spec,
7278                    runtime,
7279                    registry,
7280                    process_liveness,
7281                    snapshot,
7282                    child,
7283                    drain_timeout,
7284                )
7285                .await
7286                {
7287                    warn!(
7288                        module_id = %spec.module_id,
7289                        error = %err,
7290                        "operator restart failed after initiation ack; module state carries the outcome"
7291                    );
7292                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7293                        state.state = ModuleState::Failed;
7294                        clear_current_process_facts(state);
7295                    });
7296                } else {
7297                    runtime.start_requester = requester;
7298                    runtime.pending_activity_restart = true;
7299                }
7300            }
7301            true
7302        }
7303        SupervisorCommand::Reload { requester, reply } => {
7304            let result =
7305                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7306            if result.is_ok() {
7307                runtime.start_requester = requester;
7308                runtime.pending_activity_restart = true;
7309            }
7310            if result.is_ok()
7311                && runtime
7312                    .scheduled_respawn
7313                    .lock()
7314                    .unwrap_or_else(|p| p.into_inner())
7315                    .as_ref()
7316                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7317            {
7318                *runtime
7319                    .deferred_reload_reply
7320                    .lock()
7321                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7322            } else {
7323                let _ = reply.send(result);
7324            }
7325            true
7326        }
7327        SupervisorCommand::SetEnabled {
7328            enabled,
7329            requester,
7330            reply,
7331        } => {
7332            let previous_requester = runtime.start_requester.clone();
7333            if enabled {
7334                runtime.start_requester = requester.clone();
7335            }
7336            let result = set_child_enabled(
7337                spec,
7338                runtime,
7339                registry,
7340                process_liveness,
7341                snapshot,
7342                child,
7343                enabled,
7344            )
7345            .await;
7346            if matches!(result, Ok(true)) {
7347                runtime.pending_activity_restart = false;
7348                if enabled {
7349                    runtime.start_requester = requester;
7350                } else {
7351                    runtime.start_requester = None;
7352                    runtime.record_activity_transition(
7353                        "subc.module_disabled",
7354                        &spec.module_id,
7355                        requester,
7356                    );
7357                }
7358            } else {
7359                runtime.start_requester = previous_requester;
7360            }
7361            let _ = reply.send(result);
7362            true
7363        }
7364        SupervisorCommand::UpdateConfiguration {
7365            spec: next_spec,
7366            health,
7367            drain_timeout_ms,
7368            reply,
7369        } => {
7370            if let Some(handle) = &runtime.supervisor_handle {
7371                handle.apply_identity_configuration(&next_spec);
7372            }
7373            *spec = next_spec;
7374            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7375                state.configuration_updated_since_spawn = true;
7376            });
7377            let health_changed = runtime.health != health;
7378            runtime.health = health;
7379            // Reset the cadence and old endpoint's failure streak on a live
7380            // health-policy change rather than waiting for its old deadline.
7381            if health_changed {
7382                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7383                    state.health = ModuleHealthStatus::default();
7384                });
7385            }
7386            runtime.drain_timeout = drain_timeout_ms
7387                .map(Duration::from_millis)
7388                .unwrap_or(runtime.default_drain_timeout);
7389            *runtime
7390                .effective_drain_timeout
7391                .lock()
7392                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7393            let _ = reply.send(());
7394            true
7395        }
7396        SupervisorCommand::Swap {
7397            requester,
7398            ready_timeout,
7399            reply,
7400        } => {
7401            let mut swap_runtime = runtime.clone();
7402            swap_runtime.start_requester = requester;
7403            let before = lock_snapshot(snapshot)
7404                .ok()
7405                .map(|state| state.spawn_generation);
7406            let end = swap::run_swap(
7407                spec,
7408                &swap_runtime,
7409                registry,
7410                process_liveness,
7411                snapshot,
7412                child,
7413                commands,
7414                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7415                reply,
7416            )
7417            .await;
7418            if before
7419                != lock_snapshot(snapshot)
7420                    .ok()
7421                    .map(|state| state.spawn_generation)
7422            {
7423                runtime.start_requester = swap_runtime.start_requester;
7424            }
7425            requeued.extend(end.requeue);
7426            true
7427        }
7428    }
7429}
7430
7431async fn restart_child(
7432    spec: &ModuleSpec,
7433    runtime: &SupervisorRuntimeConfig,
7434    registry: &Registry,
7435    process_liveness: &SupervisorProcessLiveness,
7436    snapshot: &SharedSnapshot,
7437    child: &mut Option<SupervisedChild>,
7438    drain_timeout: Duration,
7439) -> Result<(), SuperviseError> {
7440    // Restart cycles a running module; it must not silently start a disabled one.
7441    if !lock_snapshot(snapshot)?.enabled {
7442        return Err(SuperviseError::Disabled {
7443            module_id: spec.module_id.clone(),
7444        });
7445    }
7446    let stop_notice = begin_forwarding_drain_with_timeout(
7447        spec,
7448        runtime,
7449        registry,
7450        snapshot,
7451        None,
7452        RouteCloseReason::Restart,
7453        drain_timeout,
7454    )
7455    .await?;
7456
7457    if child.is_some() {
7458        drain_optional_child(
7459            &spec.module_id,
7460            spec.protocol,
7461            stop_notice,
7462            registry,
7463            runtime.forwarding.as_deref(),
7464            snapshot,
7465            &runtime.terminal_ring,
7466            &runtime.spawn_events,
7467            child,
7468            drain_timeout,
7469            ModuleState::Restarting,
7470            Some(true),
7471        )
7472        .await?;
7473    } else {
7474        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7475            state.enabled = true;
7476            state.state = ModuleState::Restarting;
7477            clear_current_process_facts(state);
7478        })?;
7479        release_dead_registration(
7480            registry,
7481            runtime.forwarding.as_deref(),
7482            snapshot,
7483            &spec.module_id,
7484        )
7485        .await?;
7486    }
7487
7488    reset_restart_count(snapshot, &spec.module_id)?;
7489    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7490    schedule_respawn(
7491        runtime,
7492        snapshot,
7493        &spec.module_id,
7494        runtime.restart_policy.backoff,
7495        RespawnKind::Spawn,
7496    )
7497}
7498
7499async fn reload_child(
7500    spec: &ModuleSpec,
7501    runtime: &SupervisorRuntimeConfig,
7502    registry: &Registry,
7503    process_liveness: &SupervisorProcessLiveness,
7504    snapshot: &SharedSnapshot,
7505    child: &mut Option<SupervisedChild>,
7506) -> Result<(), SuperviseError> {
7507    // Reload cycles a running module; it must not silently start a disabled one.
7508    if !lock_snapshot(snapshot)?.enabled {
7509        return Err(SuperviseError::Disabled {
7510            module_id: spec.module_id.clone(),
7511        });
7512    }
7513    let stop_notice = begin_forwarding_drain(
7514        spec,
7515        runtime,
7516        registry,
7517        snapshot,
7518        Some(true),
7519        RouteCloseReason::Reload,
7520    )
7521    .await?;
7522
7523    if child.is_some() {
7524        drain_optional_child(
7525            &spec.module_id,
7526            spec.protocol,
7527            stop_notice,
7528            registry,
7529            runtime.forwarding.as_deref(),
7530            snapshot,
7531            &runtime.terminal_ring,
7532            &runtime.spawn_events,
7533            child,
7534            runtime.drain_timeout,
7535            ModuleState::Restarting,
7536            Some(true),
7537        )
7538        .await?;
7539    } else {
7540        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7541            state.enabled = true;
7542            state.state = ModuleState::Restarting;
7543            clear_current_process_facts(state);
7544        })?;
7545        release_dead_registration(
7546            registry,
7547            runtime.forwarding.as_deref(),
7548            snapshot,
7549            &spec.module_id,
7550        )
7551        .await?;
7552    }
7553
7554    reset_restart_count(snapshot, &spec.module_id)?;
7555    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7556    schedule_respawn(
7557        runtime,
7558        snapshot,
7559        &spec.module_id,
7560        runtime.restart_policy.backoff,
7561        RespawnKind::Reload,
7562    )
7563}
7564
7565async fn finish_reload_child(
7566    spec: &ModuleSpec,
7567    runtime: &SupervisorRuntimeConfig,
7568    registry: &Registry,
7569    process_liveness: &SupervisorProcessLiveness,
7570    snapshot: &SharedSnapshot,
7571    child: &mut Option<SupervisedChild>,
7572) -> Result<(), SuperviseError> {
7573    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7574    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7575        Ok(next_child) => next_child,
7576        Err(err) => {
7577            return handle_reload_spawn_failure(
7578                spec,
7579                runtime,
7580                process_liveness,
7581                snapshot,
7582                child,
7583                format!("new child failed to spawn: {err}"),
7584            )
7585            .await;
7586        }
7587    };
7588    *child = Some(next_child);
7589
7590    let wait_outcome = {
7591        let active_child = child.as_mut().expect("new reload child was just stored");
7592        wait_for_registration_after_reload(
7593            registry,
7594            &spec.module_id,
7595            snapshot,
7596            active_child,
7597            REGISTRY_RELEASE_TIMEOUT,
7598        )
7599        .await?
7600    };
7601
7602    match wait_outcome {
7603        RegistrationWaitOutcome::Registered => {
7604            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7605            Ok(())
7606        }
7607        RegistrationWaitOutcome::Exited(exit_report) => {
7608            if let Some(active_child) = child.as_mut() {
7609                active_child.drain_stderr(&spec.module_id).await;
7610            }
7611            // Keep the reaped child's roster guard until its terminal is written.
7612            // Shutdown waits on that guard, not on the child Option used for respawn.
7613            let mut exited_child = child.take().expect("exited reload child is still stored");
7614            #[cfg(test)]
7615            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7616                gate.reached.notify_one();
7617                gate.resume.notified().await;
7618            }
7619            let result = handle_reload_child_registration_failure(
7620                spec,
7621                runtime,
7622                registry,
7623                process_liveness,
7624                snapshot,
7625                child,
7626                ReloadRegistrationFailure {
7627                    exit_report: registration_failure_exit_report(exit_report),
7628                    reason: exited_child
7629                        .spawn_failure
7630                        .clone()
7631                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7632                },
7633            )
7634            .await;
7635            exited_child.release_roster();
7636            result
7637        }
7638        RegistrationWaitOutcome::TimedOut => {
7639            let mut timed_out_child = child
7640                .take()
7641                .expect("timed-out reload child is still running");
7642            timed_out_child
7643                .start_kill()
7644                .map_err(|source| SuperviseError::Kill {
7645                    module_id: spec.module_id.clone(),
7646                    source,
7647                })?;
7648            let status = timed_out_child
7649                .wait()
7650                .await
7651                .map_err(|source| SuperviseError::Wait {
7652                    module_id: spec.module_id.clone(),
7653                    source,
7654                })?;
7655            timed_out_child.drain_stderr(&spec.module_id).await;
7656            handle_reload_child_registration_failure(
7657                spec,
7658                runtime,
7659                registry,
7660                process_liveness,
7661                snapshot,
7662                child,
7663                ReloadRegistrationFailure {
7664                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7665                        snapshot,
7666                        &timed_out_child,
7667                        &status,
7668                    )),
7669                    reason: format!(
7670                        "new child did not register within {:?}",
7671                        REGISTRY_RELEASE_TIMEOUT
7672                    ),
7673                },
7674            )
7675            .await
7676        }
7677    }
7678}
7679
7680async fn set_child_enabled(
7681    spec: &ModuleSpec,
7682    runtime: &SupervisorRuntimeConfig,
7683    registry: &Registry,
7684    process_liveness: &SupervisorProcessLiveness,
7685    snapshot: &SharedSnapshot,
7686    child: &mut Option<SupervisedChild>,
7687    enabled: bool,
7688) -> Result<bool, SuperviseError> {
7689    let (current_enabled, current_state, respawn_pending) = {
7690        let state = lock_snapshot(snapshot)?;
7691        (state.enabled, state.state, state.respawn_pending)
7692    };
7693    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7694    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7695    // clean (Stopped) has no live process and no other in-band recovery — the
7696    // operator's start is the explicit recovery act and resets the budget. Without
7697    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7698    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7699    // the one providing every agent's shell.
7700    let revive_terminal = enabled
7701        && current_enabled
7702        && child.is_none()
7703        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7704            || (current_state == ModuleState::Restarting && !respawn_pending));
7705    if current_enabled == enabled && !revive_terminal {
7706        return Ok(false);
7707    }
7708
7709    if enabled {
7710        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7711            state.enabled = true;
7712            state.state = ModuleState::Starting;
7713            clear_current_process_facts(state);
7714        })?;
7715        #[cfg(test)]
7716        if runtime.test_seed_stale_facts_before_enable_spawn {
7717            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7718                state.process_alive = true;
7719                state.pid = Some(41);
7720                state.spawned_at_ms = Some(42);
7721                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7722                state.spawned_file_identity = Some(SpawnedFileIdentity {
7723                    device: 43,
7724                    inode: 44,
7725                    #[cfg(windows)]
7726                    windows_image: None,
7727                });
7728            })?;
7729        }
7730        release_dead_registration(
7731            registry,
7732            runtime.forwarding.as_deref(),
7733            snapshot,
7734            &spec.module_id,
7735        )
7736        .await?;
7737        reset_restart_count(snapshot, &spec.module_id)?;
7738        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7739        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7740            Ok(next_child) => next_child,
7741            Err(err) => {
7742                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7743                    state.state = ModuleState::Failed;
7744                    clear_current_process_facts(state);
7745                }) {
7746                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7747                }
7748                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7749                return Err(err);
7750            }
7751        };
7752        *child = Some(next_child);
7753        debug!(module_id = %spec.module_id, "supervised module enabled");
7754        Ok(true)
7755    } else {
7756        let stop_notice = begin_forwarding_drain_if_configured(
7757            spec,
7758            runtime,
7759            registry,
7760            snapshot,
7761            Some(false),
7762            RouteCloseReason::Disable,
7763        )
7764        .await?;
7765        drain_optional_child(
7766            &spec.module_id,
7767            spec.protocol,
7768            stop_notice,
7769            registry,
7770            runtime.forwarding.as_deref(),
7771            snapshot,
7772            &runtime.terminal_ring,
7773            &runtime.spawn_events,
7774            child,
7775            runtime.drain_timeout,
7776            ModuleState::Disabled,
7777            Some(false),
7778        )
7779        .await?;
7780        debug!(module_id = %spec.module_id, "supervised module disabled");
7781        Ok(true)
7782    }
7783}
7784
7785#[allow(clippy::too_many_arguments)]
7786async fn on_child_exit(
7787    spec: &ModuleSpec,
7788    policy: RestartPolicy,
7789    registry: &Registry,
7790    snapshot: &SharedSnapshot,
7791    terminal_ring: &Arc<Mutex<TerminalRing>>,
7792    spawn_events: &SpawnEventFeed,
7793    roster: &ChildRoster,
7794    exit_report: ExitReport,
7795) -> NextAction {
7796    // Once the daemon has begun shutting down, no exit is a crash to recover
7797    // from: the module is exiting because the daemon is going away (EOF on its
7798    // connection, or a service manager signalling the whole cgroup). Record it
7799    // as such and never schedule a respawn, which would only start a process
7800    // for the shutdown to end again.
7801    if roster.is_closed() {
7802        return on_child_exit_during_daemon_shutdown(
7803            spec,
7804            registry,
7805            snapshot,
7806            terminal_ring,
7807            spawn_events,
7808            exit_report,
7809        )
7810        .await;
7811    }
7812    // Every stop the supervisor itself asks for (operator stop, disable,
7813    // restart, reload, swap, a health restart, a drain that runs out of budget)
7814    // takes the child out of the supervise loop and reaps it in
7815    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7816    // that reaches this point was not requested by the daemon.
7817    //
7818    // For a subc-wire module a clean exit is still a stop: those modules are
7819    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7820    // crash, and exiting 0 is a deliberate choice the module made. A
7821    // `protocol: "none"` module is a stock program we cannot change, and many
7822    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7823    // stop would leave the module down for good after any stray signal, so it
7824    // goes through the crash path instead: it spends restart budget, respawns
7825    // with the crash backoff, and ends `failed` when the budget runs out.
7826    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7827        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7828    match exit_report.kind {
7829        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7830            info!(
7831                module_id = %spec.module_id,
7832                exit_code = ?exit_report.code,
7833                exit_signal = ?exit_report.signal,
7834                "supervised module exited cleanly"
7835            );
7836            if let Err(err) = publish_after_terminal_record(
7837                &spec.module_id,
7838                || {
7839                    record_terminal(
7840                        &spec.module_id,
7841                        terminal_ring,
7842                        spawn_events,
7843                        &exit_report,
7844                        TerminalDisposition::Stopped,
7845                    )
7846                },
7847                || {
7848                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7849                        state.state = ModuleState::Stopped;
7850                        clear_current_process_facts(state);
7851                        state.last_exit = Some(exit_report.clone());
7852                    })
7853                },
7854            ) {
7855                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7856            }
7857            let registration_released = match wait_for_registration_release(
7858                registry,
7859                &spec.module_id,
7860                REGISTRY_RELEASE_TIMEOUT,
7861            )
7862            .await
7863            {
7864                Ok(()) => true,
7865                Err(err) => {
7866                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7867                    false
7868                }
7869            };
7870            NextAction::Stop {
7871                registration_released,
7872            }
7873        }
7874        ExitKind::Clean | ExitKind::Crash => {
7875            if unrequested_clean_exit_of_protocol_none {
7876                warn!(
7877                    module_id = %spec.module_id,
7878                    exit_code = ?exit_report.code,
7879                    exit_signal = ?exit_report.signal,
7880                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7881                );
7882            } else {
7883                warn!(
7884                    module_id = %spec.module_id,
7885                    exit_code = ?exit_report.code,
7886                    exit_signal = ?exit_report.signal,
7887                    "supervised module exited abnormally (crash)"
7888                );
7889            }
7890            let mut restart_schedule = None;
7891            let mut disposition = TerminalDisposition::Disabled;
7892            let mut final_state = ModuleState::Disabled;
7893            // Set only when the budget is what stopped the module, so the
7894            // terminal record says which limit was hit rather than leaving
7895            // `failed` to be read as "crashed once, badly".
7896            let mut disposition_detail = lock_snapshot(snapshot)
7897                .ok()
7898                .and_then(|mut state| state.spawn_failure.take());
7899            let now = Instant::now();
7900            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7901                if state.enabled {
7902                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7903                        // Reserving the last slot exhausts the budget for the
7904                        // replacement, not for this exit. Publish its scheduled
7905                        // recovery atomically with the reservation so connection
7906                        // cleanup sees it while terminal recording still holds
7907                        // the visible state at Running.
7908                        state.respawn_pending = true;
7909                        final_state = ModuleState::Restarting;
7910                        restart_schedule = Some(schedule);
7911                        disposition = TerminalDisposition::Restarting;
7912                    } else {
7913                        final_state = ModuleState::Failed;
7914                        disposition = TerminalDisposition::Failed;
7915                        let budget = policy.budget_exhausted_detail();
7916                        disposition_detail =
7917                            Some(disposition_detail.take().map_or_else(
7918                                || budget.clone(),
7919                                |cause| format!("{cause}; {budget}"),
7920                            ));
7921                    }
7922                } else {
7923                    disposition = TerminalDisposition::Disabled;
7924                }
7925            }) {
7926                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7927                return NextAction::Stop {
7928                    registration_released: false,
7929                };
7930            }
7931            if disposition == TerminalDisposition::Failed {
7932                // The window is in the message, not only in the fields: this line
7933                // is read in a scrollback where a bare `max_restarts=3` reads as a
7934                // lifetime cap and sends the operator looking for three crashes
7935                // that never happened together.
7936                error!(
7937                    module_id = %spec.module_id,
7938                    max_restarts = policy.max_restarts,
7939                    window_secs = policy.window.as_secs(),
7940                    "module stopped: {}",
7941                    policy.budget_exhausted_detail()
7942                );
7943            }
7944            if let Err(err) = publish_after_terminal_record(
7945                &spec.module_id,
7946                || {
7947                    record_terminal_with_detail(
7948                        &spec.module_id,
7949                        terminal_ring,
7950                        spawn_events,
7951                        &exit_report,
7952                        disposition,
7953                        disposition_detail,
7954                    )
7955                },
7956                || {
7957                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7958                        state.state = final_state;
7959                        clear_current_process_facts(state);
7960                        state.last_exit = Some(exit_report.clone());
7961                    })
7962                },
7963            ) {
7964                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7965            }
7966
7967            if let Some(schedule) = restart_schedule {
7968                NextAction::Restart {
7969                    schedule: Some(schedule),
7970                }
7971            } else {
7972                let registration_released = match wait_for_registration_release(
7973                    registry,
7974                    &spec.module_id,
7975                    REGISTRY_RELEASE_TIMEOUT,
7976                )
7977                .await
7978                {
7979                    Ok(()) => true,
7980                    Err(err) => {
7981                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7982                        false
7983                    }
7984                };
7985                NextAction::Stop {
7986                    registration_released,
7987                }
7988            }
7989        }
7990        ExitKind::DeliberateSeverance => {
7991            warn!(
7992                module_id = %spec.module_id,
7993                exit_code = ?exit_report.code,
7994                exit_signal = ?exit_report.signal,
7995                "supervised module exited after deliberate connection severance"
7996            );
7997            let should_restart = match lock_snapshot(snapshot) {
7998                Ok(state) => state.enabled,
7999                Err(err) => {
8000                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8001                    return NextAction::Stop {
8002                        registration_released: false,
8003                    };
8004                }
8005            };
8006            let final_state = if should_restart {
8007                ModuleState::Restarting
8008            } else {
8009                ModuleState::Disabled
8010            };
8011            if let Err(err) = publish_after_terminal_record(
8012                &spec.module_id,
8013                || {
8014                    record_terminal(
8015                        &spec.module_id,
8016                        terminal_ring,
8017                        spawn_events,
8018                        &exit_report,
8019                        terminal_disposition(final_state),
8020                    )
8021                },
8022                || {
8023                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8024                        state.state = final_state;
8025                        clear_current_process_facts(state);
8026                        state.last_exit = Some(exit_report.clone());
8027                        state.lifetime_restarts += 1;
8028                    })
8029                },
8030            ) {
8031                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8032                return NextAction::Stop {
8033                    registration_released: false,
8034                };
8035            }
8036
8037            if should_restart {
8038                NextAction::Restart { schedule: None }
8039            } else {
8040                let registration_released = match wait_for_registration_release(
8041                    registry,
8042                    &spec.module_id,
8043                    REGISTRY_RELEASE_TIMEOUT,
8044                )
8045                .await
8046                {
8047                    Ok(()) => true,
8048                    Err(err) => {
8049                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8050                        false
8051                    }
8052                };
8053                NextAction::Stop {
8054                    registration_released,
8055                }
8056            }
8057        }
8058    }
8059}
8060
8061async fn on_child_exit_during_daemon_shutdown(
8062    spec: &ModuleSpec,
8063    registry: &Registry,
8064    snapshot: &SharedSnapshot,
8065    terminal_ring: &Arc<Mutex<TerminalRing>>,
8066    spawn_events: &SpawnEventFeed,
8067    exit_report: ExitReport,
8068) -> NextAction {
8069    info!(
8070        module_id = %spec.module_id,
8071        exit_code = ?exit_report.code,
8072        exit_signal = ?exit_report.signal,
8073        exit_kind = ?exit_report.kind,
8074        "supervised module exited during daemon shutdown; not restarting it"
8075    );
8076    if let Err(err) = publish_after_terminal_record(
8077        &spec.module_id,
8078        || {
8079            record_terminal(
8080                &spec.module_id,
8081                terminal_ring,
8082                spawn_events,
8083                &exit_report,
8084                TerminalDisposition::DaemonShutdown,
8085            )
8086        },
8087        || {
8088            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8089                state.state = ModuleState::Stopped;
8090                clear_current_process_facts(state);
8091                state.last_exit = Some(exit_report.clone());
8092            })
8093        },
8094    ) {
8095        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8096    }
8097    let registration_released =
8098        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8099            .await
8100            .is_ok();
8101    NextAction::Stop {
8102        registration_released,
8103    }
8104}
8105
8106// Keep the status lock out of terminal I/O. Readers may observe the previous
8107// state while recording is in progress, but never a completed exit without its
8108// history. Journal errors are logged by the journal and retained in the ring;
8109// a recording panic still publishes the final state before resuming the unwind.
8110fn publish_after_terminal_record<T>(
8111    module_id: &str,
8112    record: impl FnOnce(),
8113    publish: impl FnOnce() -> T,
8114) -> T {
8115    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8116    let published = publish();
8117    if let Err(panic) = recorded {
8118        error!(
8119            module_id,
8120            "terminal recording panicked; published final state before resuming unwind"
8121        );
8122        std::panic::resume_unwind(panic);
8123    }
8124    published
8125}
8126
8127fn fail_after_wait_error(
8128    module_id: &str,
8129    snapshot: &SharedSnapshot,
8130    terminal_ring: &Arc<Mutex<TerminalRing>>,
8131    spawn_events: &SpawnEventFeed,
8132    detail: String,
8133) {
8134    let report = wait_error_exit_report();
8135    publish_after_terminal_record(
8136        module_id,
8137        || {
8138            record_terminal_with_detail(
8139                module_id,
8140                terminal_ring,
8141                spawn_events,
8142                &report,
8143                TerminalDisposition::Failed,
8144                Some(detail),
8145            )
8146        },
8147        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
8148    );
8149}
8150
8151#[cfg(test)]
8152fn record_wait_error_terminal(
8153    module_id: &str,
8154    terminal_ring: &Arc<Mutex<TerminalRing>>,
8155    spawn_events: &SpawnEventFeed,
8156) {
8157    record_terminal(
8158        module_id,
8159        terminal_ring,
8160        spawn_events,
8161        &wait_error_exit_report(),
8162        TerminalDisposition::Failed,
8163    );
8164}
8165
8166fn record_terminal(
8167    module_id: &str,
8168    terminal_ring: &Arc<Mutex<TerminalRing>>,
8169    spawn_events: &SpawnEventFeed,
8170    exit_report: &ExitReport,
8171    disposition: TerminalDisposition,
8172) {
8173    record_terminal_with_detail(
8174        module_id,
8175        terminal_ring,
8176        spawn_events,
8177        exit_report,
8178        disposition,
8179        None,
8180    );
8181}
8182
8183/// The ring lock is held only to capture the read (see
8184/// `TerminalJournal::capture_read`), so this module's exits keep recording
8185/// while the journal files are read. Blocking: it reads files.
8186fn durable_terminal_history_of(
8187    terminal_ring: &Mutex<TerminalRing>,
8188    module_id: &str,
8189) -> subc_control::TerminalHistory {
8190    let read = terminal_ring
8191        .lock()
8192        .unwrap_or_else(|p| p.into_inner())
8193        .capture_durable_history();
8194    read.read(module_id)
8195}
8196
8197fn record_terminal_with_detail(
8198    module_id: &str,
8199    terminal_ring: &Arc<Mutex<TerminalRing>>,
8200    spawn_events: &SpawnEventFeed,
8201    exit_report: &ExitReport,
8202    disposition: TerminalDisposition,
8203    disposition_detail: Option<String>,
8204) {
8205    #[cfg(test)]
8206    terminal_history_tests::before_terminal_record(spawn_events);
8207    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8208    let record = TerminalRecord {
8209        exit_code: exit_report.code,
8210        exit_signal: exit_report.signal,
8211        at_ms: exit_report.at_ms,
8212        disposition,
8213        exit_kind: exit_report.kind.into(),
8214        disposition_detail,
8215    };
8216    terminal_ring
8217        .lock()
8218        .unwrap_or_else(|poisoned| poisoned.into_inner())
8219        .record_exit(module_id, record);
8220}
8221
8222fn untrack_if_registration_released(
8223    process_liveness: &SupervisorProcessLiveness,
8224    registry: &Registry,
8225    module_id: &str,
8226    snapshot: &SharedSnapshot,
8227) {
8228    match registry.get_module(module_id) {
8229        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8230        Ok(Some(_)) => {}
8231        Err(err) => {
8232            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8233        }
8234    }
8235}
8236
8237/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8238/// then apply the module's configured entries minus daemon-private capture keys.
8239///
8240/// Separated from `spawn_child` only so it can be asserted without spawning a
8241/// process — a duplicate of this logic in a test would pass while the real one
8242/// drifted, which is the defect class this function exists to avoid.
8243/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8244/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8245/// either and the argument would stop a stock binary from starting at all.
8246/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8247///
8248/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8249/// through [`apply_wire_spawn_args_for_role`].
8250#[cfg(test)]
8251fn apply_wire_spawn_args(
8252    command: &mut Command,
8253    spec: &ModuleSpec,
8254    connection_file_path: Option<&std::path::Path>,
8255    handle: Option<&SupervisorHandle>,
8256) -> Result<Option<NonceHandoff>, SuperviseError> {
8257    apply_wire_spawn_args_for_role(
8258        command,
8259        spec,
8260        connection_file_path,
8261        handle,
8262        SpawnRole::Plain,
8263    )
8264}
8265
8266/// The read end of a spawn's launch-nonce pipe, prepared by
8267/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8268/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8269/// server before spawn and authenticates its client after the child exists.
8270#[cfg(unix)]
8271type NonceHandoff = subc_os::LaunchNonceHandoff;
8272#[cfg(windows)]
8273type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8274#[cfg(not(any(unix, windows)))]
8275type NonceHandoff = std::convert::Infallible;
8276
8277/// Create the launch nonce that a new module process must present when it
8278/// registers or opens a consumer connection, including a replacement process
8279/// started alongside a still-serving module.
8280///
8281/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8282/// a separate candidate token so the still-serving incumbent and its consumers
8283/// keep their nonce. Both records are installed before the process exists, so
8284/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8285///
8286/// On Unix the nonce is delivered only through a pipe. It is written into
8287/// a pipe whose read end the child gets as descriptor 3, named by
8288/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8289/// process of the same user cannot read it with `ps eww`. That handoff is
8290/// returned rather than installed here, because installing it replaces
8291/// whatever the child has at descriptor 3 and so must be the last pre-exec
8292/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8293/// the Linux callback that writes the process into its cgroup. Only after that
8294/// does `handoff.install_last` register the descriptor-3 replacement, so it
8295/// cannot close a descriptor the cgroup callback still needs.
8296/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8297fn apply_wire_spawn_args_for_role(
8298    command: &mut Command,
8299    spec: &ModuleSpec,
8300    connection_file_path: Option<&std::path::Path>,
8301    handle: Option<&SupervisorHandle>,
8302    role: SpawnRole,
8303) -> Result<Option<NonceHandoff>, SuperviseError> {
8304    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8305    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8306    // included: a daemon started from a module's process tree inherits it,
8307    // and passing it on would point the child at a descriptor it does not
8308    // have.
8309    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8310    #[cfg(windows)]
8311    {
8312        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8313        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8314    }
8315    // Remove inherited or configured copies too: withholding must mean absent.
8316    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8317    if spec.protocol == ModuleProtocol::None {
8318        return Ok(None);
8319    }
8320    if let Some(connection_file_path) = connection_file_path {
8321        command.arg(SUBC_ARG).arg(connection_file_path);
8322    }
8323
8324    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8325    // route.open attestation. Reserved modules additionally use the same nonce
8326    // for HELLO id-squatting protection. A respawn rotates both records.
8327    let nonce = generate_launch_nonce()?;
8328    if let Some(handle) = handle {
8329        match role {
8330            SpawnRole::Plain => {
8331                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8332                if spec.reserved {
8333                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8334                }
8335            }
8336            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8337        }
8338    }
8339    #[cfg(unix)]
8340    let handoff = {
8341        let handoff =
8342            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8343                program: spec.program.clone(),
8344                source,
8345                cgroup_path: None,
8346            })?;
8347        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8348        Some(handoff)
8349    };
8350    #[cfg(windows)]
8351    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8352    #[cfg(not(any(unix, windows)))]
8353    let handoff = None;
8354    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8355    // until the daemon's per-spawn delivery records show that every running
8356    // Windows module has read its secret from the named pipe.
8357    #[cfg(not(unix))]
8358    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8359    Ok(handoff)
8360}
8361
8362fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8363    command.env_remove(CK_LOG_ENV);
8364    // The spawn role is the supervisor's to set, and only on a swap candidate
8365    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8366    // it, is what makes it absent on a plain spawn: the daemon's own
8367    // environment could carry it, and so could a spec built outside daemon
8368    // config (config refuses it as an `env` key). A module reading it on a
8369    // plain restart would pick the long swap budget and leave callers waiting.
8370    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8371    for (key, value) in &spec.env {
8372        // cortexkit-log currently exposes retention only as a Rust struct, not
8373        // environment names. These values are daemon-private sink metadata and
8374        // must never become a public child-process contract by being inherited.
8375        if matches!(
8376            key.as_str(),
8377            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8378        ) || key == SUBC_SPAWN_ROLE_ENV
8379        {
8380            continue;
8381        }
8382        command.env(key, value);
8383    }
8384}
8385
8386#[cfg(windows)]
8387fn prepare_windows_nonce_pipe(
8388    command: &mut Command,
8389    spec: &ModuleSpec,
8390    nonce: &str,
8391) -> Option<NonceHandoff> {
8392    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8393        Ok(handoff) => {
8394            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8395            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8396            // still register until every module reads the pipe. A reader without
8397            // this variable set to env returns the pipe error instead of using
8398            // an inherited or configured environment copy.
8399            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8400            Some(handoff)
8401        }
8402        Err(error) => {
8403            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8404            None
8405        }
8406    }
8407}
8408
8409#[cfg(windows)]
8410fn start_windows_nonce_pipe(
8411    handoff: Option<NonceHandoff>,
8412    pid: u32,
8413    spec: &ModuleSpec,
8414) -> Option<subc_os::LaunchNoncePipeDelivery> {
8415    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8416    // same maximum wait used for registration during a reload. Older modules
8417    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8418    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8419        Ok(delivery) => Some(delivery),
8420        Err(error) => {
8421            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8422            None
8423        }
8424    })
8425}
8426
8427fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8428    #[cfg(windows)]
8429    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8430        return Some(
8431            if snapshot
8432                .pipe_consumed
8433                .as_ref()
8434                .is_some_and(|consumed| consumed.get().is_some())
8435            {
8436                "pipe"
8437            } else {
8438                "env"
8439            }
8440            .to_owned(),
8441        );
8442    }
8443    let _ = snapshot;
8444    None
8445}
8446
8447/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8448/// of a blue/green swap.
8449#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8450enum SpawnRole {
8451    Plain,
8452    SwapCandidate,
8453}
8454
8455/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8456/// `apply_child_env` has already removed the variable for every spawn.
8457fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8458    if role == SpawnRole::SwapCandidate {
8459        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8460    }
8461}
8462
8463fn previous_instance_error(
8464    spec: &ModuleSpec,
8465    pid: u32,
8466    reason: crate::live_children::RetainedReason,
8467) -> SuperviseError {
8468    SuperviseError::Spawn {
8469        program: spec.program.clone(),
8470        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8471        cgroup_path: None,
8472    }
8473}
8474
8475fn spawn_child(
8476    spec: &ModuleSpec,
8477    connection_file_path: Option<&std::path::Path>,
8478    handle: Option<&SupervisorHandle>,
8479    ring: &Arc<Mutex<StderrRing>>,
8480    capture_logs_dir: Option<&std::path::Path>,
8481    roster: &ChildRoster,
8482    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8483) -> Result<SupervisedChild, SuperviseError> {
8484    spawn_child_in_slot(
8485        spec,
8486        connection_file_path,
8487        handle,
8488        ring,
8489        capture_logs_dir,
8490        roster,
8491        #[cfg(target_os = "linux")]
8492        cgroup_placement,
8493        SpawnRole::Plain,
8494        false,
8495    )
8496}
8497
8498/// Spawn one process of `spec` into a slot.
8499///
8500/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8501/// A swap candidate needs a different cgroup from the process it is replacing,
8502/// which is still alive: in the same cgroup the two would be one kill domain,
8503/// and killing a failed candidate could take the incumbent with it.
8504///
8505/// The stderr capture file is `<module_id>.stderr.log` for every process of
8506/// the module, whichever slot it is in, because that is the one file
8507/// `ck module logs` reads. During a swap's overlap both processes append to it;
8508/// the daemon writes whole lines, so the two interleave by line, which is also
8509/// the merged view an operator wants while a swap runs.
8510#[allow(clippy::too_many_arguments)]
8511fn spawn_child_in_slot(
8512    spec: &ModuleSpec,
8513    connection_file_path: Option<&std::path::Path>,
8514    handle: Option<&SupervisorHandle>,
8515    ring: &Arc<Mutex<StderrRing>>,
8516    capture_logs_dir: Option<&std::path::Path>,
8517    roster: &ChildRoster,
8518    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8519    role: SpawnRole,
8520    alternate_slot: bool,
8521) -> Result<SupervisedChild, SuperviseError> {
8522    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8523        return Err(previous_instance_error(spec, pid, reason));
8524    }
8525    if roster.is_closed() {
8526        return Err(SuperviseError::Spawn {
8527            program: spec.program.clone(),
8528            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8529            cgroup_path: None,
8530        });
8531    }
8532    #[cfg(target_os = "linux")]
8533    let cgroup_name = {
8534        // Slot names alone are not kill domains: a retired incumbent may still
8535        // be draining when a later enable/restart spawns into the same slot.
8536        // Decimal entropy keeps the suffix unambiguous; Placement performs
8537        // the module-id escaping and constructs the filesystem path.
8538        if cgroup_placement.is_none() {
8539            swap::cgroup_name(&spec.module_id, alternate_slot)
8540        } else {
8541            let nonce = generate_launch_nonce()?;
8542            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8543            // Leave room for byte escaping and the suffix under NAME_MAX. The
8544            // label is only for humans; the nonce identifies the kill domain.
8545            let mut end = spec.module_id.len().min(64);
8546            while !spec.module_id.is_char_boundary(end) {
8547                end -= 1;
8548            }
8549            format!(
8550                "{}_{suffix}",
8551                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8552            )
8553        }
8554    };
8555    #[cfg(not(target_os = "linux"))]
8556    let _ = alternate_slot;
8557    #[cfg(target_os = "macos")]
8558    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8559    #[cfg(windows)]
8560    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8561        program: spec.program.clone(),
8562        source,
8563        cgroup_path: None,
8564    })?;
8565    #[cfg(windows)]
8566    let mut command = Command::new(image_capture.path());
8567    #[cfg(not(any(target_os = "macos", windows)))]
8568    let mut command = Command::new(&spec.program);
8569    command.args(&spec.args);
8570    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8571    // that is the whole of the intent, so remove that one key rather than the
8572    // environment.
8573    //
8574    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8575    // and took the POSIX environment with it. Modules spawned that way had no
8576    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8577    // logging:
8578    //
8579    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8580    //     both unset it fell back to the temp dir alone and `ck` could not find
8581    //     a daemon running on the same machine from inside any module's process
8582    //     tree — reporting a path the file has never lived at, which reads as
8583    //     "the daemon did not write its file".
8584    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8585    //     the RELATIVE `.local/share`, so a module deriving its own store path
8586    //     resolved it against its own CWD. That is the store-fragmentation
8587    //     defect the daemon already refuses in config (`parse_doc` rejects a
8588    //     relative `storage.data_home`) arriving by derivation instead.
8589    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8590    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8591    //     quietly rather than erroring.
8592    //
8593    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8594    // offered one candidate under /tmp while the file sat in /run/user/1000.
8595    //
8596    // A configured module is unaffected either way: `module_spec()` puts the
8597    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8598    // wins over anything ambient.
8599    apply_child_env(&mut command, spec);
8600    apply_spawn_role(&mut command, role);
8601    let nonce_handoff =
8602        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8603
8604    #[cfg(target_os = "linux")]
8605    let cgroup_path = cgroup_placement
8606        .map(|placement| placement.module_path(&cgroup_name))
8607        .transpose()
8608        .map_err(|source| SuperviseError::Cgroup {
8609            module_id: spec.module_id.clone(),
8610            source,
8611        })?;
8612    #[cfg(not(target_os = "linux"))]
8613    let cgroup_path: Option<PathBuf> = None;
8614    #[cfg(target_os = "linux")]
8615    if let Some(path) = &cgroup_path {
8616        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8617            if let Some(placement) = cgroup_placement {
8618                remove_module_cgroup(placement, &cgroup_name);
8619            }
8620            return Err(error);
8621        }
8622    }
8623
8624    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8625        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8626        match ChildOutputSink::open(&path, capture_retention(spec)) {
8627            Ok(sink) => sink,
8628            Err(error) => {
8629                warn!(
8630                    module_id = %spec.module_id,
8631                    path = %path.display(),
8632                    error = %error,
8633                    "could not open child output capture file; forwarding to stderr"
8634                );
8635                ChildOutputSink::Stderr
8636            }
8637        }
8638    } else {
8639        ChildOutputSink::Stderr
8640    };
8641
8642    command.stdout(Stdio::piped());
8643    command.stderr(Stdio::piped());
8644    command.kill_on_drop(true);
8645    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8646    // before exec). In the daemon's group, a service manager that kills the
8647    // job's process group when the daemon exits (launchd's default) killed
8648    // every module at the same moment its control connection closed, so no
8649    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8650    // module is reached only by the daemon: the EOF it sees when its
8651    // connection closes, and the bounded stop in `child_roster` for anything
8652    // still running after that. On Linux this composes with the cgroup
8653    // placement above: that is a pre_exec write to cgroup.procs, std performs
8654    // setpgid in the child before running pre_exec callbacks, and the two
8655    // change independent process attributes.
8656    //
8657    // stdin is /dev/null because a process outside the terminal's foreground
8658    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8659    // by hand would otherwise hand down. Under a service manager stdin is
8660    // already /dev/null.
8661    #[cfg(unix)]
8662    command.process_group(0);
8663    command.stdin(Stdio::null());
8664    // The LAST pre-exec step, after the cgroup placement above: installing the
8665    // nonce at descriptor 3 replaces whatever the child had there, which could
8666    // be the descriptor an earlier step writes through.
8667    #[cfg(unix)]
8668    if let Some(handoff) = nonce_handoff {
8669        handoff.install_last(command.as_std_mut());
8670    }
8671    #[cfg(not(any(unix, windows)))]
8672    let _ = nonce_handoff;
8673
8674    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8675    // cannot run a single instruction -- and therefore cannot spawn a
8676    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8677    // other two steps and why the window matters.
8678    #[cfg(windows)]
8679    {
8680        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8681            program: spec.program.clone(),
8682            source,
8683            cgroup_path: None,
8684        })?;
8685        subc_jobobject::suspend_on_create_async(&mut command);
8686    }
8687    // Timestamp before the spawn call: on Unix the child can serve before that
8688    // call even returns. This conservative bound never labels an older child
8689    // as a replacement for a request received during process creation.
8690    let spawned_at = Instant::now();
8691    let mut child = match command.spawn() {
8692        Ok(child) => child,
8693        Err(source) => {
8694            #[cfg(target_os = "linux")]
8695            if let Some(placement) = cgroup_placement {
8696                remove_module_cgroup(placement, &cgroup_name);
8697            }
8698            return Err(SuperviseError::Spawn {
8699                program: spec.program.clone(),
8700                source,
8701                cgroup_path,
8702            });
8703        }
8704    };
8705    #[cfg(test)]
8706    spawn_record_tests::after_create(&child, spec);
8707    // The parent must close its writer now: the acknowledgement pipe reports EOF
8708    // only when every writer is gone, and the child's copy closes when the
8709    // trampoline replaces itself with the module. Command holds only an integer
8710    // in its pre_exec callback, not another writer.
8711    #[cfg(target_os = "macos")]
8712    drop(exec_ack);
8713
8714    // Bind while the child is still suspended, before job assignment resumes it.
8715    // Keeping the executable handle open across CreateProcessW denies rename
8716    // and deletion, so the file cannot be exchanged during spawn. Bind that
8717    // file ID to the process handle returned by spawn, never a reopened PID.
8718    #[cfg(windows)]
8719    let windows_image = match image_capture.bind_tokio(&child) {
8720        Ok(image) => Arc::new(image),
8721        Err(source) => {
8722            kill_and_reap_rejected_child(&mut child);
8723            return Err(SuperviseError::Spawn {
8724                program: spec.program.clone(),
8725                source,
8726                cgroup_path: None,
8727            });
8728        }
8729    };
8730    // Assign the suspended child to its module job; resume only after its
8731    // process identity has been durably recorded for orphan recovery.
8732    #[cfg(windows)]
8733    let job = contain_spawned_child(&mut child, spec)?;
8734    let spawned_at_ms = unix_ms_now();
8735    #[cfg(not(windows))]
8736    let spawned_from = spec.program.clone();
8737    #[cfg(windows)]
8738    let spawned_from = windows_image.path().to_path_buf();
8739    #[cfg(not(windows))]
8740    let spawned_file_identity = spawned_file_identity(&spawned_from);
8741    #[cfg(windows)]
8742    let spawned_file_identity = Some(SpawnedFileIdentity {
8743        device: 0,
8744        inode: 0,
8745        windows_image: Some(Arc::clone(&windows_image)),
8746    });
8747    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8748        program: spec.program.clone(),
8749        source: io::Error::other("spawned child exposed no live pid"),
8750        cgroup_path: cgroup_path.clone(),
8751    })?;
8752    #[cfg(windows)]
8753    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8754    #[cfg(not(windows))]
8755    let process_start_time = crate::provenance::process_start_time(pid);
8756    #[cfg(windows)]
8757    let process_start_time = Some(windows_image.start_time());
8758    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8759    #[cfg(all(test, target_os = "macos"))]
8760    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8761    // Record the committed kernel image, not the configured pathname: PATH
8762    // lookup and shebang interpretation can select a different executable.
8763    // Linux's posix_spawn can return before that image is installed, so the
8764    // observer also waits for the kernel's first-exec flag to clear.
8765    #[cfg(not(windows))]
8766    let recorded_image = observe_spawned_image(pid);
8767    // spawn() confirms only the first exec, into the trampoline. Never persist
8768    // the trampoline image; the asynchronous acknowledgement publishes the
8769    // module image once the trampoline has replaced itself with the module.
8770    #[cfg(target_os = "macos")]
8771    let recorded_image = if privacy_exec.is_some() {
8772        None
8773    } else {
8774        recorded_image
8775    };
8776    #[cfg(target_os = "linux")]
8777    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8778    #[cfg(not(target_os = "linux"))]
8779    let recorded_cgroup_name = None;
8780    let roster_guard = roster.admit(
8781        spec.module_id.clone(),
8782        pid,
8783        spec.protocol,
8784        process_start_time,
8785        crate::child_roster::RecordedIdentity {
8786            #[cfg(not(windows))]
8787            start_time: recorded_image.map(|image| image.start_time),
8788            #[cfg(windows)]
8789            start_time: process_start_time,
8790            #[cfg(not(windows))]
8791            executable: recorded_image
8792                .and_then(|image| image.executable)
8793                .map(crate::live_children::ExecutableIdentity::from),
8794            #[cfg(windows)]
8795            executable: Some(crate::live_children::ExecutableIdentity::from(
8796                windows_image.identity(),
8797            )),
8798            cgroup_name: recorded_cgroup_name,
8799            #[cfg(target_os = "linux")]
8800            cgroup_placement: cgroup_placement.cloned(),
8801        },
8802    );
8803    let roster_guard = match roster_guard {
8804        Ok(guard) => guard,
8805        Err(failure) => {
8806            return failure.with_cleanup(|source| {
8807                #[cfg(windows)]
8808                if let Some(job) = &job {
8809                    let _ = job.terminate();
8810                }
8811                #[cfg(target_os = "linux")]
8812                kill_module_cgroup(cgroup_placement, &cgroup_name);
8813                kill_and_reap_rejected_child(&mut child);
8814                #[cfg(target_os = "linux")]
8815                if let Some(placement) = cgroup_placement {
8816                    remove_module_cgroup(placement, &cgroup_name);
8817                }
8818                Err(SuperviseError::Spawn {
8819                    program: spec.program.clone(),
8820                    source: io::Error::other(format!(
8821                        "could not durably record module child: {source}"
8822                    )),
8823                    cgroup_path,
8824                })
8825            });
8826        }
8827    };
8828    // The pre-spawn check can pass just before shutdown begins. Ownership is
8829    // published at the start of admission, even while a record write is queued.
8830    // If shutdown already saw an empty roster before that publication, this
8831    // check sees the roster closed and ends the child before starting a module.
8832    if roster.is_closed() {
8833        // This child was never admitted, so there is no module protocol shutdown to wait for.
8834        #[cfg(target_os = "linux")]
8835        kill_module_cgroup(cgroup_placement, &cgroup_name);
8836        if let Err(error) = child.start_kill() {
8837            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8838        }
8839        #[cfg(target_os = "linux")]
8840        if let Some(placement) = cgroup_placement {
8841            // This spawn was never admitted, so shutdown has no roster entry
8842            // to await. Do not detach its cleanup: the runtime could exit
8843            // before that task reaps the rejected child and removes its group.
8844            while matches!(child.try_wait(), Ok(None)) {
8845                std::thread::yield_now();
8846            }
8847            if matches!(
8848                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8849                subc_cgroup::KillOutcome::Killed
8850            ) {
8851                if let Ok(path) = placement.module_path(&cgroup_name) {
8852                    while std::fs::read_to_string(path.join("cgroup.events"))
8853                        .ok()
8854                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8855                    {
8856                        std::thread::yield_now();
8857                    }
8858                }
8859            }
8860            remove_module_cgroup(placement, &cgroup_name);
8861        }
8862        drop(roster_guard);
8863        return Err(SuperviseError::Spawn {
8864            program: spec.program.clone(),
8865            source: io::Error::other(
8866                "the daemon began shutting down while this process was starting; ended it",
8867            ),
8868            cgroup_path,
8869        });
8870    }
8871
8872    #[cfg(windows)]
8873    if let Err(error) = resume_suspended_child(pid, spec) {
8874        kill_and_reap_rejected_child(&mut child);
8875        return Err(error);
8876    }
8877
8878    let stdout_pump = match child.stdout.take() {
8879        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8880        None => {
8881            warn!(
8882                module_id = %spec.module_id,
8883                "spawned child exposed no stdout pipe; file capture will be incomplete"
8884            );
8885            None
8886        }
8887    };
8888    let stderr_pump = match child.stderr.take() {
8889        Some(stderr) => {
8890            let generation = ring
8891                .lock()
8892                .unwrap_or_else(|poisoned| poisoned.into_inner())
8893                .begin_process();
8894            Some(StderrPump {
8895                task: tokio::spawn(pump_stderr_to(
8896                    stderr,
8897                    Arc::clone(ring),
8898                    generation,
8899                    output_sink,
8900                )),
8901                generation,
8902            })
8903        }
8904        None => {
8905            // Spawning succeeded but the pipe did not materialise. Recording it as
8906            // uncaptured keeps the tail honest: the alternative is an empty tail
8907            // that reads as a module which printed nothing.
8908            ring.lock()
8909                .unwrap_or_else(|poisoned| poisoned.into_inner())
8910                .mark_not_captured("stderr pipe was not available on spawn");
8911            warn!(
8912                module_id = %spec.module_id,
8913                "spawned child exposed no stderr pipe; tail will be unavailable"
8914            );
8915            None
8916        }
8917    };
8918
8919    let exec = Arc::new(crate::activity::ExecOutcome::new());
8920    #[cfg(target_os = "macos")]
8921    let pending_exec = privacy_exec.is_some();
8922    #[cfg(not(target_os = "macos"))]
8923    let pending_exec = false;
8924    if !pending_exec {
8925        exec.finish(crate::activity::ExecState::CommittedExec);
8926    }
8927    Ok(SupervisedChild {
8928        child,
8929        exec: crate::activity::ExecOwner(exec),
8930        #[cfg(windows)]
8931        nonce_delivery,
8932        protocol: spec.protocol,
8933        #[cfg(target_os = "linux")]
8934        module_id: cgroup_name,
8935        #[cfg(target_os = "linux")]
8936        cgroup_placement: cgroup_placement.cloned(),
8937        #[cfg(windows)]
8938        job,
8939        stdout_pump,
8940        stderr_pump,
8941        stderr_ring: Arc::clone(ring),
8942        spawned_at,
8943        spawned_at_ms,
8944        spawned_from,
8945        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8946        spawned_file_identity,
8947        process_start_time,
8948        process_identity,
8949        pid,
8950        roster_guard: Some(roster_guard),
8951        #[cfg(target_os = "macos")]
8952        privacy_exec,
8953        #[cfg(target_os = "macos")]
8954        report_ready: Arc::new(OnceLock::new()),
8955        spawn_failure: None,
8956    })
8957}
8958
8959#[cfg(test)]
8960mod spawn_record_tests {
8961    use super::*;
8962
8963    thread_local! {
8964        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8965    }
8966
8967    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8968        LAST_PID.set(child.id());
8969        #[cfg(windows)]
8970        if let Some((_, address)) = _spec
8971            .env
8972            .iter()
8973            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8974        {
8975            use std::io::{Read, Write};
8976            let mut stream = std::net::TcpStream::connect(address).unwrap();
8977            stream
8978                .write_all(&child.id().unwrap().to_ne_bytes())
8979                .unwrap();
8980            let mut release = [0];
8981            stream.read_exact(&mut release).unwrap();
8982        }
8983    }
8984
8985    #[cfg(windows)]
8986    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8987        if spec
8988            .env
8989            .iter()
8990            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8991        {
8992            Err(io::Error::other("injected module job assignment failure"))
8993        } else {
8994            Ok(())
8995        }
8996    }
8997
8998    fn spec() -> ModuleSpec {
8999        ModuleSpec {
9000            module_id: "spawn-record-worker".into(),
9001            #[cfg(unix)]
9002            program: "/bin/sleep".into(),
9003            #[cfg(windows)]
9004            program: std::env::current_exe().unwrap(),
9005            #[cfg(unix)]
9006            args: vec!["60".into()],
9007            #[cfg(windows)]
9008            args: vec![
9009                "--exact".into(),
9010                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9011                "--nocapture".into(),
9012            ],
9013            env: vec![],
9014            reserved: false,
9015            reserved_prefixes: vec![],
9016            protocol: ModuleProtocol::None,
9017            overlap: Default::default(),
9018        }
9019    }
9020
9021    #[tokio::test]
9022    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9023        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9024        let supervisor =
9025            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9026                .with_live_children_record(home.join("live-children.json"));
9027        crate::live_children::fail_next_file_sync();
9028        let result = supervisor.spawn(spec());
9029        let failed = result.is_err();
9030        if let Ok(module) = result {
9031            module.stop().await.unwrap();
9032        }
9033        assert!(failed, "record failure must fail spawn");
9034        let pid = LAST_PID.get().expect("the real OS child was created");
9035        assert!(
9036            subc_os::Process::open(pid).unwrap().is_none(),
9037            "child {pid} survived rejected admission or was not reaped"
9038        );
9039    }
9040
9041    #[cfg(target_os = "macos")]
9042    #[test]
9043    #[ignore = "manual on-device latency measurement"]
9044    fn durable_record_write_latency_50() {
9045        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9046        let path = home.join("live-children.json");
9047        let children = vec![crate::live_children::LiveChild {
9048            module_id: "latency-worker".into(),
9049            pid: std::process::id(),
9050            protocol: ModuleProtocol::None,
9051            start_time: Some(1),
9052            executable: None,
9053            cgroup_name: None,
9054        }];
9055        let mut samples = Vec::new();
9056        for _ in 0..50 {
9057            let start = std::time::Instant::now();
9058            crate::live_children::write_record(&path, &children).unwrap();
9059            samples.push(start.elapsed());
9060        }
9061        samples.sort();
9062        eprintln!(
9063            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9064            (samples[24] + samples[25]) / 2,
9065            samples[47]
9066        );
9067    }
9068
9069    #[cfg(windows)]
9070    #[test]
9071    fn windows_store_touch_fixture() {
9072        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9073            std::fs::write(path, b"module ran").unwrap();
9074        }
9075    }
9076
9077    #[cfg(windows)]
9078    #[tokio::test]
9079    async fn windows_daemon_fixture() {
9080        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9081            return;
9082        };
9083        let mut spec = spec();
9084        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9085        let supervisor =
9086            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9087        let _module = supervisor.spawn(spec).unwrap();
9088    }
9089
9090    #[cfg(windows)]
9091    #[test]
9092    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9093        use std::io::Read;
9094        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9095        // An outer job models a scheduler/service host. The daemon's root and
9096        // root job must nest without requiring breakaway from this job.
9097        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9098        command
9099            .args([
9100                "--exact",
9101                "supervise::spawn_record_tests::windows_daemon_fixture",
9102                "--nocapture",
9103            ])
9104            .env(
9105                "SUBC_TEST_DAEMON_BARRIER",
9106                listener.local_addr().unwrap().to_string(),
9107            );
9108        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9109        listener.set_nonblocking(true).unwrap();
9110        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9111        let mut stream = loop {
9112            if let Ok((stream, _)) = listener.accept() {
9113                break stream;
9114            }
9115            assert!(
9116                daemon.child.try_wait().unwrap().is_none(),
9117                "daemon failed before rendezvous"
9118            );
9119            assert!(
9120                std::time::Instant::now() < deadline,
9121                "daemon did not reach CreateProcess failpoint"
9122            );
9123            std::thread::yield_now();
9124        };
9125        stream
9126            .set_read_timeout(Some(Duration::from_secs(30)))
9127            .unwrap();
9128        let mut pid = [0; 4];
9129        stream.read_exact(&mut pid).unwrap();
9130        let pid = u32::from_ne_bytes(pid);
9131        assert!(subc_jobobject::process_exists(pid));
9132        // Terminate only the daemon, not its outer job: otherwise the host job
9133        // would also kill a child whose daemon never joined its root job.
9134        daemon.child.kill().unwrap();
9135        daemon.child.wait().unwrap();
9136        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9137        if !exited {
9138            daemon.job.terminate().unwrap();
9139        }
9140        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9141    }
9142
9143    #[cfg(windows)]
9144    #[tokio::test]
9145    async fn windows_containment_failure_never_resumes() {
9146        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9147        let sentinel = home.join("store-touched");
9148        let mut spec = spec();
9149        spec.env
9150            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9151        spec.env.push((
9152            "SUBC_TEST_STORE_TOUCH".into(),
9153            sentinel.display().to_string(),
9154        ));
9155        let supervisor =
9156            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9157        let result = supervisor.spawn(spec);
9158        // A version that resumes despite assignment failure must have time to
9159        // execute its sentinel before dropping the returned module kills it.
9160        if let Ok(module) = result {
9161            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9162                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9163                    tokio::task::yield_now().await;
9164                }
9165            })
9166            .await;
9167            let _ = module.stop().await;
9168            panic!(
9169                "assignment failure resumed a module; store touched={}",
9170                sentinel.exists()
9171            );
9172        }
9173        assert!(
9174            !sentinel.exists(),
9175            "refused module executed store-touch sentinel"
9176        );
9177        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9178    }
9179
9180    #[cfg(windows)]
9181    #[tokio::test]
9182    async fn windows_nested_daemon_fixture() {
9183        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9184            return;
9185        };
9186        let mut spec = spec();
9187        spec.env.push((
9188            "SUBC_TEST_STORE_TOUCH".into(),
9189            sentinel.to_str().unwrap().into(),
9190        ));
9191        let supervisor =
9192            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9193        let module = supervisor
9194            .spawn(spec)
9195            .expect("per-module job must nest below daemon and host jobs");
9196        tokio::time::timeout(Duration::from_secs(10), async {
9197            while !PathBuf::from(&sentinel).exists() {
9198                tokio::task::yield_now().await;
9199            }
9200        })
9201        .await
9202        .unwrap();
9203        module.stop().await.unwrap();
9204    }
9205
9206    #[cfg(windows)]
9207    #[test]
9208    fn windows_module_job_nests_under_host_job() {
9209        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9210        let sentinel = home.join("nested-store-touched");
9211        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9212        command
9213            .args([
9214                "--exact",
9215                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9216                "--nocapture",
9217            ])
9218            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9219        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9220        assert!(daemon.child.wait().unwrap().success());
9221        assert!(sentinel.exists(), "nested protocol:none module never ran");
9222    }
9223}
9224
9225#[cfg(target_os = "linux")]
9226pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9227    use subc_cgroup::KillOutcome;
9228    match subc_cgroup::kill_module(placement, module_id) {
9229        KillOutcome::Killed => {}
9230        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9231            debug!(
9232                module_id,
9233                "cgroup tree kill unavailable; using direct-child kill"
9234            );
9235        }
9236        KillOutcome::IoError { path, error } => {
9237            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9238        }
9239    }
9240}
9241
9242/// Assign a freshly spawned Windows child without ever resuming on failure.
9243///
9244/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9245/// child assigned **while it is still suspended** (step 1 is
9246/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9247/// the child's roster admission is durable. Failed containment kills and reaps
9248/// the suspended child instead of running an unprotected module.
9249#[cfg(windows)]
9250fn contain_spawned_child(
9251    child: &mut Child,
9252    spec: &ModuleSpec,
9253) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9254    let assignment = || {
9255        let job = subc_jobobject::JobObject::new()?;
9256        #[cfg(test)]
9257        spawn_record_tests::assignment_failpoint(spec)?;
9258        job.assign(child)?;
9259        Ok::<_, io::Error>(job)
9260    };
9261    match assignment() {
9262        Ok(job) => Ok(Some(job)),
9263        Err(source) => {
9264            kill_and_reap_rejected_child(child);
9265            Err(SuperviseError::Spawn {
9266                program: spec.program.clone(),
9267                source: io::Error::other(format!(
9268                    "Windows module job containment failed: {source}"
9269                )),
9270                cgroup_path: None,
9271            })
9272        }
9273    }
9274}
9275
9276/// Rejection is synchronous: do not return a spawn error while the child can
9277/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9278fn kill_and_reap_rejected_child(child: &mut Child) {
9279    #[cfg(unix)]
9280    if let Some(pid) = child
9281        .id()
9282        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9283    {
9284        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9285    }
9286    let _ = child.start_kill();
9287    while matches!(child.try_wait(), Ok(None)) {
9288        std::thread::yield_now();
9289    }
9290}
9291
9292/// Resume a suspended child, killing it if it cannot be started.
9293///
9294/// A suspended process holds a pid and does nothing, so there is no useful
9295/// state to return: the caller gets an error and the spawn fails.
9296#[cfg(windows)]
9297fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9298    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9299        // Kill it here rather than leaving a suspended process for the caller
9300        // to notice; `kill_on_drop` would eventually do this, but the module
9301        // would have been reported as running in between.
9302        let _ = std::process::Command::new("taskkill.exe")
9303            .args(["/PID", &pid.to_string(), "/T", "/F"])
9304            .stdin(Stdio::null())
9305            .stdout(Stdio::null())
9306            .stderr(Stdio::null())
9307            .status();
9308        return Err(SuperviseError::Spawn {
9309            program: spec.program.clone(),
9310            source,
9311            cgroup_path: None,
9312        });
9313    }
9314    Ok(())
9315}
9316
9317#[cfg(target_os = "linux")]
9318fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9319    match placement.remove_module(module_id) {
9320        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9321        Err(error) => warn!(
9322            module_id,
9323            error = %error,
9324            "could not remove module cgroup after process exit; continuing teardown"
9325        ),
9326    }
9327}
9328
9329#[cfg(target_os = "linux")]
9330async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9331    // Reaping the direct child is not proof its descendants exited. End the
9332    // residual tree and wait for the kernel's population fact before rmdir;
9333    // otherwise a successful parent wait leaks a directory on each restart.
9334    if matches!(
9335        subc_cgroup::kill_module(Some(placement), module_id),
9336        subc_cgroup::KillOutcome::Killed
9337    ) {
9338        if let Ok(path) = placement.module_path(module_id) {
9339            while std::fs::read_to_string(path.join("cgroup.events"))
9340                .ok()
9341                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9342            {
9343                sleep(Duration::from_millis(1)).await;
9344            }
9345        }
9346    }
9347    remove_module_cgroup(placement, module_id);
9348}
9349
9350#[cfg(target_os = "linux")]
9351fn apply_cgroup_placement(
9352    command: &mut Command,
9353    spec: &ModuleSpec,
9354    path: &std::path::Path,
9355) -> Result<(), SuperviseError> {
9356    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9357        module_id: spec.module_id.clone(),
9358        source,
9359    })
9360}
9361
9362fn capture_retention(spec: &ModuleSpec) -> Retention {
9363    let defaults = Retention::default();
9364    let value = |name: &str| {
9365        spec.env
9366            .iter()
9367            .rev()
9368            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9369    };
9370    Retention {
9371        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9372            .and_then(|value| value.parse().ok())
9373            .unwrap_or(defaults.max_file_mb),
9374        keep: value(CAPTURE_KEEP_ENV)
9375            .and_then(|value| value.parse().ok())
9376            .unwrap_or(defaults.keep),
9377        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9378            .and_then(|value| value.parse().ok())
9379            .unwrap_or(defaults.max_age_days),
9380    }
9381}
9382
9383/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9384/// module's registration to the exact process the supervisor spawned.
9385fn generate_launch_nonce() -> Result<String, SuperviseError> {
9386    let mut bytes = [0u8; 32];
9387    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9388        reason: source.to_string(),
9389    })?;
9390    let mut hex = String::with_capacity(64);
9391    for b in bytes {
9392        use std::fmt::Write;
9393        let _ = write!(hex, "{b:02x}");
9394    }
9395    Ok(hex)
9396}
9397
9398/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9399/// signal about how many leading bytes matched.
9400fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9401    if a.len() != b.len() {
9402        return false;
9403    }
9404    let mut diff = 0u8;
9405    for (x, y) in a.iter().zip(b.iter()) {
9406        diff |= x ^ y;
9407    }
9408    diff == 0
9409}
9410
9411/// The kernel's image after an acknowledged exec, shared by ordinary launches
9412/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9413/// not identities inferred from a configured pathname.
9414#[cfg(not(windows))]
9415fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9416    let process = subc_os::Process::open(pid).ok().flatten()?;
9417    #[cfg(target_os = "linux")]
9418    return sample_after_linux_exec(
9419        || {
9420            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9421            let committed = linux_exec_committed(&stat);
9422            #[cfg(test)]
9423            spawn_image_tests::report_exec_state(committed);
9424            committed
9425        },
9426        || process.observe(),
9427    );
9428    #[cfg(not(target_os = "linux"))]
9429    process.observe()
9430}
9431
9432/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9433/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9434/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9435#[cfg(target_os = "linux")]
9436fn linux_exec_committed(stat: &str) -> Option<bool> {
9437    const PF_FORKNOEXEC: u64 = 0x40;
9438    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9439    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9440}
9441
9442#[cfg(target_os = "linux")]
9443fn sample_after_linux_exec(
9444    mut exec_committed: impl FnMut() -> Option<bool>,
9445    observe: impl FnOnce() -> Option<subc_os::Observation>,
9446) -> Option<subc_os::Observation> {
9447    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9448    while !exec_committed()? {
9449        // A failed or indefinitely stalled exec must not publish the inherited
9450        // image as authority to signal a future orphan. Missing identity keeps
9451        // the existing conservative orphan-sweep behavior.
9452        if std::time::Instant::now() >= deadline {
9453            return None;
9454        }
9455        std::thread::sleep(Duration::from_micros(50));
9456    }
9457    observe()
9458}
9459
9460#[cfg(all(test, target_os = "linux"))]
9461mod spawn_image_tests {
9462    use super::*;
9463    use std::cell::{Cell, RefCell};
9464
9465    enum ExecEvent {
9466        Uncommitted,
9467        Image(Option<subc_os::Observation>),
9468    }
9469
9470    thread_local! {
9471        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9472    }
9473
9474    pub(super) fn report_exec_state(committed: Option<bool>) {
9475        if committed == Some(false) {
9476            EXEC_EVENTS.with(|events| {
9477                if let Some(events) = events.borrow_mut().take() {
9478                    let _ = events.send(ExecEvent::Uncommitted);
9479                }
9480            });
9481        }
9482    }
9483
9484    #[test]
9485    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9486        use std::io::{Read, Write};
9487        use std::os::fd::AsRawFd;
9488
9489        let descriptor = std::fs::File::open("/dev/null").unwrap();
9490        let mut command = std::process::Command::new("/bin/sleep");
9491        command.arg("30");
9492        let (mut ready, mut release) =
9493            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9494                .unwrap();
9495        let spawn = std::thread::spawn(move || command.spawn());
9496        let mut message = [0u8; 8];
9497        ready.read_exact(&mut message).unwrap();
9498        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9499        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9500        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9501        let (events, received) = std::sync::mpsc::channel();
9502        let observer = std::thread::spawn(move || {
9503            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9504            let image = observe_spawned_image(pid);
9505            let _ = events.send(ExecEvent::Image(image));
9506        });
9507        // Release only once the production observer has seen the uncommitted
9508        // exec, or has incorrectly returned the inherited image. Either way,
9509        // release and reap before asserting so a regression cannot leak a child.
9510        let first = received.recv_timeout(Duration::from_secs(5));
9511        release.write_all(&[1]).unwrap();
9512        let mut child = spawn.join().unwrap().unwrap();
9513        let after = match first {
9514            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9515                Ok(ExecEvent::Image(image)) => image,
9516                _ => None,
9517            },
9518            Ok(ExecEvent::Image(image)) => image,
9519            Err(_) => None,
9520        };
9521        observer.join().unwrap();
9522        child.kill().unwrap();
9523        child.wait().unwrap();
9524
9525        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9526        assert_eq!(
9527            inherited.unwrap().executable,
9528            subc_os::file_identity(&std::env::current_exe().unwrap())
9529        );
9530        assert_eq!(
9531            after.unwrap().executable,
9532            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9533        );
9534    }
9535
9536    #[test]
9537    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9538        let committed = Cell::new(false);
9539        let inherited = subc_os::Observation {
9540            start_time: 23,
9541            executable: Some(subc_os::FileIdentity {
9542                device: 29,
9543                inode: 31,
9544            }),
9545        };
9546        let module = subc_os::Observation {
9547            start_time: 23,
9548            executable: Some(subc_os::FileIdentity {
9549                device: 29,
9550                inode: 37,
9551            }),
9552        };
9553        let polls = Cell::new(0);
9554        let image = sample_after_linux_exec(
9555            || {
9556                polls.set(polls.get() + 1);
9557                // Hold the inherited image through the first poll, as in the
9558                // posix_spawn parent-wake window, then commit the module image.
9559                committed.set(polls.get() == 2);
9560                Some(committed.get())
9561            },
9562            || Some(if committed.get() { module } else { inherited }),
9563        );
9564        assert_eq!(image, Some(module));
9565        assert_eq!(polls.get(), 2);
9566    }
9567
9568    #[test]
9569    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9570        assert_eq!(
9571            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9572            Some(false)
9573        );
9574        assert_eq!(
9575            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9576            Some(true)
9577        );
9578        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9579        assert_eq!(linux_exec_committed("17 (a) R"), None);
9580    }
9581
9582    #[test]
9583    fn unreadable_exec_state_never_samples_an_inherited_image() {
9584        assert_eq!(
9585            sample_after_linux_exec(
9586                || None,
9587                || panic!("must not sample before exec confirmation")
9588            ),
9589            None
9590        );
9591    }
9592
9593    #[test]
9594    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9595        assert_eq!(
9596            sample_after_linux_exec(
9597                || Some(false),
9598                || panic!("must not sample after an exec timeout")
9599            ),
9600            None
9601        );
9602    }
9603}
9604
9605/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9606/// different executable after the no-delete capture was taken.
9607#[cfg(windows)]
9608fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9609    let mut program = spec.program.clone();
9610    if program.is_absolute() || program.components().count() > 1 {
9611        // An explicit path can name a PE image with no extension. Try that
9612        // exact file first, as Command does, before falling back to .exe.
9613        match subc_os::ExecutableCapture::open(&program) {
9614            Ok(capture) => return Ok(capture),
9615            Err(error)
9616                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9617            Err(error) => return Err(error),
9618        }
9619        program.set_extension("exe");
9620        return subc_os::ExecutableCapture::open(&program);
9621    }
9622    if program.extension().is_none() {
9623        program.set_extension("exe");
9624    }
9625    let custom_path = spec
9626        .env
9627        .iter()
9628        .rev()
9629        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9630    let mut directories = Vec::new();
9631    // These are Windows' default executable search locations. A configured
9632    // PATH still overrides the inherited PATH, not the system directories.
9633    if let Some(parent) = std::env::current_exe()?.parent() {
9634        directories.push(parent.to_path_buf());
9635    }
9636    directories.push(std::env::current_dir()?);
9637    if let Some(windows) = std::env::var_os("SystemRoot") {
9638        let windows = PathBuf::from(windows);
9639        directories.push(windows.join("System32"));
9640        directories.push(windows.clone());
9641    }
9642    let path = custom_path
9643        .map(|(_, value)| std::ffi::OsString::from(value))
9644        .or_else(|| std::env::var_os("PATH"));
9645    if let Some(path) = path {
9646        directories.extend(std::env::split_paths(&path));
9647    }
9648    for directory in directories {
9649        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9650            Ok(capture) => return Ok(capture),
9651            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9652            Err(error) => return Err(error),
9653        }
9654    }
9655    Err(io::Error::new(
9656        io::ErrorKind::NotFound,
9657        "resolved executable not found",
9658    ))
9659}
9660
9661fn spawn_and_mark_running(
9662    spec: &ModuleSpec,
9663    runtime: &SupervisorRuntimeConfig,
9664    snapshot: &SharedSnapshot,
9665) -> Result<SupervisedChild, SuperviseError> {
9666    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9667    let child = spawn_child(
9668        spec,
9669        runtime.connection_file_path.as_deref(),
9670        runtime.supervisor_handle.as_ref(),
9671        &runtime.stderr_ring,
9672        runtime.capture_logs_dir.as_deref(),
9673        &runtime.child_roster,
9674        #[cfg(target_os = "linux")]
9675        runtime.cgroup_placement.as_ref(),
9676    )?;
9677    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9678    if let Some(spawn) = activity_spawn {
9679        spawn.complete(snapshot);
9680    }
9681    Ok(child)
9682}
9683
9684enum RegistrationWaitOutcome {
9685    Registered,
9686    Exited(ExitReport),
9687    TimedOut,
9688}
9689
9690struct ReloadRegistrationFailure {
9691    exit_report: ExitReport,
9692    reason: String,
9693}
9694
9695#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9696enum BusyGaugeObservation {
9697    Quiescent,
9698    Busy,
9699    Omitted,
9700}
9701
9702/// What one health answer said about one declared busy gauge.
9703#[derive(Debug, Clone, PartialEq, Eq)]
9704enum GaugeVerdict {
9705    /// The gauge read 0: this gauge is not what holds the drain.
9706    Zero,
9707    /// The gauge read a positive count.
9708    NonZero(u64),
9709    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9710    /// a negative number). It counts as busy, because the daemon cannot tell
9711    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9712    NotACount(String),
9713    /// The health answer had no metrics object, or one without this gauge.
9714    Missing,
9715    /// The probe produced no health answer to read; carries the probe error's
9716    /// evidence label (`no-answer` for a probe that timed out).
9717    ProbeFailed(&'static str),
9718}
9719
9720impl GaugeVerdict {
9721    fn reads_quiet(&self) -> bool {
9722        matches!(self, Self::Zero)
9723    }
9724}
9725
9726impl fmt::Display for GaugeVerdict {
9727    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9728        match self {
9729            Self::Zero => f.write_str("zero"),
9730            Self::NonZero(value) => write!(f, "non_zero({value})"),
9731            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9732            Self::Missing => f.write_str("missing_from_health_body"),
9733            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9734        }
9735    }
9736}
9737
9738/// The longest JSON text kept for a gauge value that is not a count, so a
9739/// module reporting a large object as a gauge cannot flood the drain log.
9740const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9741
9742/// Read each declared gauge out of one health answer's metrics.
9743fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9744    let metrics = metrics.and_then(Value::as_object);
9745    gauges
9746        .iter()
9747        .map(
9748            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9749                None => GaugeVerdict::Missing,
9750                Some(value) => match value.as_u64() {
9751                    Some(0) => GaugeVerdict::Zero,
9752                    Some(count) => GaugeVerdict::NonZero(count),
9753                    None => {
9754                        let mut text = value.to_string();
9755                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9756                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9757                            while !text.is_char_boundary(end) {
9758                                end -= 1;
9759                            }
9760                            text.truncate(end);
9761                            text.push_str("...");
9762                        }
9763                        GaugeVerdict::NotACount(text)
9764                    }
9765                },
9766            },
9767        )
9768        .collect()
9769}
9770
9771/// The drain's overall reading of one health answer: omitted when any
9772/// declared gauge is missing, quiescent only when every gauge reads 0.
9773fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9774    if verdicts
9775        .iter()
9776        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9777    {
9778        BusyGaugeObservation::Omitted
9779    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9780        BusyGaugeObservation::Quiescent
9781    } else {
9782        BusyGaugeObservation::Busy
9783    }
9784}
9785
9786/// One declared gauge's history over a drain: the latest verdict on it, and
9787/// the latest value actually read for it (which a later failed probe or
9788/// missing gauge does not erase). Times are offsets from the start of the
9789/// drain's wait.
9790#[derive(Debug, Clone, Default, PartialEq, Eq)]
9791struct GaugeTrace {
9792    latest: Option<(GaugeVerdict, Duration)>,
9793    last_value: Option<(GaugeVerdict, Duration)>,
9794}
9795
9796/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9797/// its ceiling can say in its timeout line which declared gauge held it and
9798/// why. Without it, a gauge that read busy, a gauge the module never
9799/// reported, and a probe that never got an answer all looked the same: a
9800/// drain that ended at its ceiling with nothing in flight.
9801#[derive(Debug, Clone, Default, PartialEq, Eq)]
9802struct DrainGaugeTrace {
9803    /// Health probes the drain sent.
9804    probes: u32,
9805    /// Each declared gauge, in declaration order.
9806    gauges: Vec<(String, GaugeTrace)>,
9807}
9808
9809impl DrainGaugeTrace {
9810    fn new(gauges: &[String]) -> Self {
9811        Self {
9812            probes: 0,
9813            gauges: gauges
9814                .iter()
9815                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9816                .collect(),
9817        }
9818    }
9819
9820    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9821    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9822        self.probes = self.probes.saturating_add(1);
9823        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9824            trace.latest = Some((verdict.clone(), at));
9825            if matches!(
9826                verdict,
9827                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9828            ) {
9829                trace.last_value = Some((verdict.clone(), at));
9830            }
9831        }
9832    }
9833}
9834
9835/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
9836/// declared gauge, separated by ` | `. A value is shown as its verdict
9837/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
9838/// and whether that counted as busy.
9839impl fmt::Display for DrainGaugeTrace {
9840    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9841        if self.gauges.is_empty() {
9842            return f.write_str("none declared");
9843        }
9844        for (index, (name, trace)) in self.gauges.iter().enumerate() {
9845            if index > 0 {
9846                f.write_str(" | ")?;
9847            }
9848            write!(f, "{name}: ")?;
9849            match &trace.latest {
9850                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9851                None => f.write_str("never probed")?,
9852            }
9853            match &trace.last_value {
9854                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9855                None => f.write_str("; no value ever read")?,
9856            }
9857        }
9858        Ok(())
9859    }
9860}
9861
9862pub(crate) fn declared_busy_gauges(
9863    registry: &Registry,
9864    module_id: &str,
9865) -> Result<Vec<String>, SuperviseError> {
9866    busy_gauges_of(
9867        registry
9868            .get_module(module_id)
9869            .map_err(SuperviseError::Registry)?,
9870    )
9871}
9872
9873/// [`declared_busy_gauges`] for the registration a connection holds, in any
9874/// slot: after cutover the incumbent is no longer the id's active
9875/// registration, and its own manifest is the one that names its gauges.
9876fn declared_busy_gauges_for_connection(
9877    registry: &Registry,
9878    connection_id: ConnectionId,
9879) -> Result<Vec<String>, SuperviseError> {
9880    busy_gauges_of(
9881        registry
9882            .get_module_by_connection(connection_id)
9883            .map_err(SuperviseError::Registry)?,
9884    )
9885}
9886
9887fn busy_gauges_of(
9888    registration: Option<crate::registry::ModuleRegistration>,
9889) -> Result<Vec<String>, SuperviseError> {
9890    let Some(registration) = registration else {
9891        return Ok(Vec::new());
9892    };
9893    let Some(self_signals) = registration.manifest.self_signals else {
9894        return Ok(Vec::new());
9895    };
9896
9897    let mut gauges = Vec::new();
9898    for declaration in self_signals {
9899        if declaration.kind != SelfSignalKind::Busy {
9900            continue;
9901        }
9902        match declaration.anchored_to {
9903            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9904                gauges.extend(declared)
9905            }
9906            _ => {
9907                // An invalid Busy anchor is fail-safe: the empty name cannot be
9908                // present in a conforming health report, so this drain stays busy.
9909                gauges.push(String::new());
9910            }
9911        }
9912    }
9913    Ok(gauges)
9914}
9915
9916/// Wait for `endpoint` to have nothing in flight and, when the module declares
9917/// busy gauges, for a health probe to report them quiet. The probe is addressed
9918/// by `scope`: a swap's superseded incumbent must be asked about its own
9919/// gauges, and by module id the probe would reach the promoted candidate.
9920/// While the gauges read busy, the probe repeats every
9921/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9922/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
9923/// timeout line.
9924#[allow(clippy::too_many_arguments)]
9925async fn wait_for_forwarding_quiescence(
9926    forwarding: &ForwardingTable,
9927    module_id: &str,
9928    runtime: &SupervisorRuntimeConfig,
9929    endpoint: crate::ModuleEndpointId,
9930    deadline: Instant,
9931    busy_gauges: &[String],
9932    scope: DrainScope,
9933    trace: &mut DrainGaugeTrace,
9934) -> Result<bool, SuperviseError> {
9935    let wait_started = Instant::now();
9936    let mut gauges_quiescent = busy_gauges.is_empty();
9937    let mut next_probe_at = wait_started;
9938    let mut omission_reported = false;
9939
9940    loop {
9941        let now = Instant::now();
9942        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9943            let report = match scope {
9944                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9945                DrainScope::Endpoint(endpoint) => {
9946                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9947                }
9948            };
9949            let verdicts = match &report {
9950                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9951                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9952            };
9953            trace.record(&verdicts, wait_started.elapsed());
9954            gauges_quiescent = match report {
9955                Ok(_) => match busy_gauge_observation(&verdicts) {
9956                    BusyGaugeObservation::Quiescent => true,
9957                    BusyGaugeObservation::Busy => false,
9958                    BusyGaugeObservation::Omitted => {
9959                        // Once per drain: a module that never reports a gauge
9960                        // it declared would otherwise say so on every probe.
9961                        if !omission_reported {
9962                            forwarding
9963                                .counters()
9964                                .increment_drains_with_undeclared_gauge();
9965                            let missing = busy_gauges
9966                                .iter()
9967                                .zip(&verdicts)
9968                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9969                                .map(|(gauge, _)| gauge.as_str())
9970                                .collect::<Vec<_>>()
9971                                .join(",");
9972                            warn!(
9973                                module_id,
9974                                missing_gauges = %missing,
9975                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9976                            );
9977                            omission_reported = true;
9978                        }
9979                        false
9980                    }
9981                },
9982                Err(err) => {
9983                    warn!(
9984                        module_id,
9985                        error = %err,
9986                        "drain health.check did not produce declared busy gauges; treating module as busy"
9987                    );
9988                    false
9989                }
9990            };
9991            // Counted from the probe's completion: the probe above was awaited,
9992            // so no second probe can start while one is still outstanding.
9993            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9994        }
9995
9996        let in_flight = forwarding
9997            .endpoint_in_flight_count(endpoint)
9998            .map_err(SuperviseError::Forwarding)?;
9999        if in_flight == 0 && gauges_quiescent {
10000            return Ok(true);
10001        }
10002
10003        let now = Instant::now();
10004        if now >= deadline {
10005            return Ok(false);
10006        }
10007        let mut wait = deadline
10008            .saturating_duration_since(now)
10009            .min(REGISTRY_RELEASE_POLL);
10010        if !busy_gauges.is_empty() {
10011            wait = wait.min(next_probe_at.saturating_duration_since(now));
10012        }
10013        sleep(wait).await;
10014    }
10015}
10016
10017/// The wait between drain busy-gauge probes: the larger of
10018/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10019/// drain's own in-flight poll interval.
10020fn drain_busy_reprobe_interval() -> Duration {
10021    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10022}
10023
10024/// Logs why a drain gave up: the drain waited its full timeout without the
10025/// module going quiet, and teardown is being forced.
10026///
10027/// The line names two kinds of evidence, because a bare "timed out" hides
10028/// causes that need different fixes:
10029/// - `holdouts`: the requests and routes still open when the timeout hit,
10030///   and on which connections. One stuck call is a module bug; a held stream
10031///   on every session is a module that should end its streams when it
10032///   receives `module.draining`.
10033/// - `gauges`: for each busy gauge the module declares, its latest reading
10034///   and how many probes ran. This separates a module that kept reporting
10035///   itself busy, one whose health answer never contained the gauge, and one
10036///   that never answered the probes, which otherwise all look like a timeout
10037///   with nothing in flight.
10038fn warn_drain_timed_out(
10039    module_id: &str,
10040    drain_timeout: Duration,
10041    reason: RouteCloseReason,
10042    holdouts: &crate::forwarding::DrainHoldouts,
10043    gauges: &DrainGaugeTrace,
10044) {
10045    warn!(
10046        module_id = %module_id,
10047        waited = ?drain_timeout,
10048        ?reason,
10049        held_requests = holdouts.requests,
10050        held_routes = holdouts.routes,
10051        total_routes = holdouts.total_routes,
10052        top_connections = ?holdouts.top_connections,
10053        // `module_channel:corr`, so the module can find each held request
10054        // in its own log; capped, so `held_requests` is the full count.
10055        held = %holdouts
10056            .held
10057            .iter()
10058            .map(|(channel, corr)| format!("{channel}:{corr}"))
10059            .collect::<Vec<_>>()
10060            .join(","),
10061        gauge_probes = gauges.probes,
10062        busy_gauges = %gauges,
10063        "route drain timed out before request quiescence; forcing teardown"
10064    );
10065}
10066
10067/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10068///
10069/// `Ok` is always honest and passed straight through -- the wait actually measured
10070/// in-flight state. `Err` means the wait produced no measurement at all (the
10071/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10072/// constant: the drain did not complete. Never recomputed from route state, never a
10073/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10074fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10075    match wait_result {
10076        Ok(drained) => *drained,
10077        Err(_) => false,
10078    }
10079}
10080
10081fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10082    for released in released_routes {
10083        let frame = match Frame::build_with_version(
10084            released.negotiated_ver,
10085            FrameType::Goodbye,
10086            control_flags(),
10087            released.channel,
10088            released.epoch,
10089            0,
10090            Vec::new(),
10091        ) {
10092            Ok(frame) => frame,
10093            Err(err) => {
10094                warn!(
10095                    route_channel = released.channel,
10096                    error = %err,
10097                    "failed to build supervisor drain route GOODBYE frame"
10098                );
10099                continue;
10100            }
10101        };
10102        if !released.close_on_delivery_failure() {
10103            crate::forwarding::send_module_route_goodbye(
10104                &forwarding.counters(),
10105                &released.sink,
10106                frame,
10107                released.module_id.as_deref(),
10108                "supervisor drain",
10109            );
10110            continue;
10111        }
10112        if let Err(err) = released.sink.try_send(frame) {
10113            warn!(
10114                target_connection_id = released.connection_id.get(),
10115                route_channel = released.channel,
10116                error = %err,
10117                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10118            );
10119            let _ = forwarding.escalate_client_delivery_failure(
10120                released.connection_id,
10121                released.channel,
10122                released.epoch,
10123                CloseReason::new(
10124                    "route_goodbye_delivery_failed",
10125                    format!(
10126                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10127                        released.channel
10128                    ),
10129                ),
10130                crate::forwarding::UndeliveredFrame {
10131                    module_id: released.module_id.as_deref(),
10132                    sink: &released.sink,
10133                },
10134            );
10135        }
10136    }
10137}
10138
10139fn send_module_draining(
10140    module_id: &str,
10141    reason: RouteCloseReason,
10142    deadline_ms: u64,
10143    target: &ModuleDrainTarget,
10144) {
10145    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10146        reason,
10147        deadline_ms,
10148    }) {
10149        Ok(body) => body,
10150        Err(err) => {
10151            warn!(
10152                module_id,
10153                error = %err,
10154                "failed to encode module draining command"
10155            );
10156            return;
10157        }
10158    };
10159    let frame = match Frame::build_with_version(
10160        target.negotiated_ver,
10161        FrameType::Push,
10162        control_flags(),
10163        0,
10164        0,
10165        0,
10166        body,
10167    ) {
10168        Ok(frame) => frame,
10169        Err(err) => {
10170            warn!(
10171                module_id,
10172                error = %err,
10173                "failed to build module draining command frame"
10174            );
10175            return;
10176        }
10177    };
10178    if let Err(err) = target.sink.try_send(frame) {
10179        warn!(
10180            module_id,
10181            target_connection_id = target.endpoint.connection_id.get(),
10182            error = %err,
10183            "module draining command was not delivered to peer"
10184        );
10185    }
10186}
10187
10188/// The channel-0 GOODBYE that tells a module its stop is planned.
10189fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10190    match Frame::build_with_version(
10191        negotiated_ver,
10192        FrameType::Goodbye,
10193        control_flags(),
10194        0,
10195        0,
10196        0,
10197        Vec::new(),
10198    ) {
10199        Ok(frame) => Some(frame),
10200        Err(err) => {
10201            warn!(
10202                module_id,
10203                error = %err,
10204                "failed to build module GOODBYE frame"
10205            );
10206            None
10207        }
10208    }
10209}
10210
10211/// Send every registered module connection its module GOODBYE at daemon
10212/// shutdown, then request that connection's close.
10213///
10214/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10215/// before EOF, so the GOODBYE must reach the socket before the close. A close
10216/// request does not wait for the connection's queued frames: its writer gets a
10217/// bounded grace after the close, is aborted if it overruns it, and the daemon
10218/// process may exit before that grace ends. So with `wait_for_flush`, each
10219/// connection is closed only after its writer has acknowledged writing the
10220/// GOODBYE, or once a short shared budget runs out, so one module that is not
10221/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10222/// are only queued, for a shutdown the operator has told to stop waiting.
10223/// A connection that is already gone is skipped.
10224#[cfg(unix)]
10225async fn send_module_goodbyes_for_daemon_shutdown(
10226    forwarding: &Arc<ForwardingTable>,
10227    reason: &CloseReason,
10228    wait_for_flush: bool,
10229) {
10230    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10231    let targets = match forwarding.module_connections() {
10232        Ok(targets) => targets,
10233        Err(err) => {
10234            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10235            return;
10236        }
10237    };
10238    let deadline = Instant::now() + GOODBYE_BUDGET;
10239    let mut sends = tokio::task::JoinSet::new();
10240    for target in targets {
10241        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10242            continue;
10243        };
10244        if !wait_for_flush {
10245            if let Err(err) = target.sink.try_send(frame) {
10246                debug!(
10247                    module_id = %target.module_id,
10248                    error = %err,
10249                    "shutdown module GOODBYE was not queued"
10250                );
10251            }
10252            continue;
10253        }
10254        let forwarding = Arc::clone(forwarding);
10255        let reason = reason.clone();
10256        sends.spawn(async move {
10257            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10258                Ok(Ok(())) => {}
10259                Ok(Err(err)) => debug!(
10260                    module_id = %target.module_id,
10261                    error = %err,
10262                    "module connection closed before its shutdown GOODBYE was written"
10263                ),
10264                Err(_) => warn!(
10265                    module_id = %target.module_id,
10266                    budget = ?GOODBYE_BUDGET,
10267                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10268                ),
10269            }
10270            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10271        });
10272    }
10273    // Every task ends by the shared deadline, so this wait is bounded too.
10274    while sends.join_next().await.is_some() {}
10275}
10276
10277fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10278    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10279        return;
10280    };
10281    if let Err(err) = target.sink.try_send(frame) {
10282        warn!(
10283            module_id,
10284            target_connection_id = target.endpoint.connection_id.get(),
10285            error = %err,
10286            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10287        );
10288        forwarding.request_connection_close(
10289            target.endpoint.connection_id,
10290            CloseReason::new(
10291                "module_goodbye_delivery_failed",
10292                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10293            ),
10294        );
10295    }
10296}
10297
10298#[derive(Clone, Copy)]
10299struct ForwardingDrainContext<'a> {
10300    spec: &'a ModuleSpec,
10301    runtime: &'a SupervisorRuntimeConfig,
10302    registry: &'a Registry,
10303    scope: DrainScope,
10304}
10305
10306/// Which process a forwarding drain addresses.
10307#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10308enum DrainScope {
10309    /// Whatever endpoint is active for the module id: every plain stop,
10310    /// restart and reload. Also moves the module's state to `Draining`.
10311    Active,
10312    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10313    /// module id would resolve to the promoted candidate and leave neither
10314    /// process routable. The module's state is left alone, since the promoted
10315    /// candidate is what it describes and that process is running.
10316    Endpoint(crate::ModuleEndpointId),
10317}
10318
10319/// Whether a child being drained has already been asked to stop by the time
10320/// its drain wait starts.
10321///
10322/// The drain wait is the same budget whatever this says. What it decides is
10323/// whether the supervisor must ask by signal before that wait begins: a child
10324/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10325/// healthy or not.
10326#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10327enum StopNotice {
10328    /// The module was sent `module.draining` and a module GOODBYE over its own
10329    /// registered connection, and stops itself.
10330    SentOverConnection,
10331    /// The forwarding drain found no registered connection for the module: a
10332    /// subc child spawned moments ago that has not sent HELLO yet, or a
10333    /// `protocol: "none"` child, which never registers.
10334    NoConnection,
10335    /// This path sends nothing over the module's connection: the supervisor has
10336    /// no forwarding table, or the caller stops the child without a forwarding
10337    /// drain.
10338    NotSent,
10339}
10340
10341async fn begin_forwarding_drain(
10342    spec: &ModuleSpec,
10343    runtime: &SupervisorRuntimeConfig,
10344    registry: &Registry,
10345    snapshot: &SharedSnapshot,
10346    enabled: Option<bool>,
10347    reason: RouteCloseReason,
10348) -> Result<StopNotice, SuperviseError> {
10349    let Some(forwarding) = runtime.forwarding.as_ref() else {
10350        return Err(SuperviseError::ReloadUnavailable {
10351            module_id: spec.module_id.clone(),
10352            reason: "supervisor was not configured with a forwarding table".to_string(),
10353        });
10354    };
10355
10356    begin_forwarding_drain_with(
10357        forwarding,
10358        ForwardingDrainContext {
10359            spec,
10360            runtime,
10361            registry,
10362            scope: DrainScope::Active,
10363        },
10364        snapshot,
10365        enabled,
10366        reason,
10367        runtime.drain_timeout,
10368    )
10369    .await
10370}
10371
10372async fn begin_forwarding_drain_if_configured(
10373    spec: &ModuleSpec,
10374    runtime: &SupervisorRuntimeConfig,
10375    registry: &Registry,
10376    snapshot: &SharedSnapshot,
10377    enabled: Option<bool>,
10378    reason: RouteCloseReason,
10379) -> Result<StopNotice, SuperviseError> {
10380    begin_forwarding_drain_with_timeout(
10381        spec,
10382        runtime,
10383        registry,
10384        snapshot,
10385        enabled,
10386        reason,
10387        runtime.drain_timeout,
10388    )
10389    .await
10390}
10391
10392/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10393/// budget, for paths where the operator overrides the module's configured one
10394/// (`supervisor.restart{drain_timeout_ms}`).
10395async fn begin_forwarding_drain_with_timeout(
10396    spec: &ModuleSpec,
10397    runtime: &SupervisorRuntimeConfig,
10398    registry: &Registry,
10399    snapshot: &SharedSnapshot,
10400    enabled: Option<bool>,
10401    reason: RouteCloseReason,
10402    drain_timeout: Duration,
10403) -> Result<StopNotice, SuperviseError> {
10404    let Some(forwarding) = runtime.forwarding.as_ref() else {
10405        return Ok(StopNotice::NotSent);
10406    };
10407
10408    begin_forwarding_drain_with(
10409        forwarding,
10410        ForwardingDrainContext {
10411            spec,
10412            runtime,
10413            registry,
10414            scope: DrainScope::Active,
10415        },
10416        snapshot,
10417        enabled,
10418        reason,
10419        drain_timeout,
10420    )
10421    .await
10422}
10423
10424async fn begin_forwarding_drain_with(
10425    forwarding: &ForwardingTable,
10426    context: ForwardingDrainContext<'_>,
10427    snapshot: &SharedSnapshot,
10428    enabled: Option<bool>,
10429    reason: RouteCloseReason,
10430    drain_timeout: Duration,
10431) -> Result<StopNotice, SuperviseError> {
10432    let ForwardingDrainContext {
10433        spec,
10434        runtime,
10435        registry,
10436        scope,
10437    } = context;
10438    debug_assert_ne!(reason, RouteCloseReason::Crash);
10439    let terminal = matches!(reason, RouteCloseReason::Disable);
10440    let drain_started_at = Instant::now();
10441    let drain_deadline = drain_started_at + drain_timeout;
10442    let deadline_ms =
10443        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10444    let busy_gauges = match scope {
10445        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10446        DrainScope::Endpoint(endpoint) => {
10447            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10448        }
10449    };
10450
10451    // Admission gate first: route.open/commit and route REQUEST admission are closed
10452    // before the first quiescence check, so the outstanding count can only fall.
10453    let gate_started = Instant::now();
10454    let drain_target = match scope {
10455        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10456        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10457    }
10458    .map_err(SuperviseError::Forwarding)?;
10459    // The instant admission closed, and how long taking the forwarding write
10460    // lock to close it took. The timeout line reports only the quiescence
10461    // wait, so without this a drain that started late looked like one that
10462    // started on time.
10463    info!(
10464        module_id = %spec.module_id,
10465        ?reason,
10466        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10467        connected = drain_target.is_some(),
10468        "module drain began; route admission closed"
10469    );
10470    if scope == DrainScope::Active {
10471        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10472            state.state = ModuleState::Draining;
10473            state.draining_to_replace =
10474                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10475            if let Some(enabled) = enabled {
10476                state.enabled = enabled;
10477            }
10478        })?;
10479    }
10480
10481    let Some(target) = drain_target.as_ref() else {
10482        // Nothing was sent: the module has no registered connection to carry
10483        // `module.draining` or a GOODBYE. The caller must not assume the child
10484        // was asked to stop.
10485        return Ok(StopNotice::NoConnection);
10486    };
10487    {
10488        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10489        #[cfg(feature = "test-support")]
10490        forwarding.wait_before_route_closing_for_test().await;
10491        let routes = forwarding
10492            .endpoint_routes(target.endpoint)
10493            .map_err(SuperviseError::Forwarding)?;
10494        let routes_notified = routes.len();
10495        crate::control::send_route_control_pushes(
10496            forwarding,
10497            routes.clone(),
10498            ClientControlPush::RouteClosing {
10499                module_id: spec.module_id.clone(),
10500                channels: Vec::new(),
10501                reason,
10502            },
10503        );
10504        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10505
10506        // `route.closing` was just sent above: from here on every return path,
10507        // including an early one, MUST send `route.closed` before propagating
10508        // anything else. A client holds `closing` as a promise that a verdict is
10509        // coming; leaving early without `closed` strands it waiting forever, since
10510        // `closing` carries no timeout of its own.
10511        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10512        let wait_result = wait_for_forwarding_quiescence(
10513            forwarding,
10514            &spec.module_id,
10515            runtime,
10516            target.endpoint,
10517            drain_deadline,
10518            &busy_gauges,
10519            scope,
10520            &mut gauge_trace,
10521        )
10522        .await;
10523        let drained = drained_after_quiescence_wait(&wait_result);
10524        if let Err(err) = &wait_result {
10525            error!(
10526                module_id = %spec.module_id,
10527                ?reason,
10528                error = %err,
10529                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10530            );
10531        } else if !drained {
10532            // Read before teardown releases the routes.
10533            let holdouts = forwarding
10534                .endpoint_drain_holdouts(target.endpoint)
10535                .unwrap_or_default();
10536            warn_drain_timed_out(
10537                &spec.module_id,
10538                drain_timeout,
10539                reason,
10540                &holdouts,
10541                &gauge_trace,
10542            );
10543        }
10544        crate::control::send_route_control_pushes(
10545            forwarding,
10546            routes,
10547            ClientControlPush::RouteClosed {
10548                module_id: spec.module_id.clone(),
10549                channels: Vec::new(),
10550                reason,
10551                drained,
10552                abandoned: target.abandoned_bindings.len() as u32,
10553                excluded_subscriptions: target.excluded_subscriptions,
10554                terminal: Some(terminal),
10555            },
10556        );
10557        wait_result?;
10558
10559        // `route.closed` has now been sent unconditionally above. From here the
10560        // remaining steps are cleanup (route + module GOODBYE) rather than a
10561        // promise the client is waiting on, but a lock-poisoned
10562        // `release_module_endpoint_routes` would otherwise skip the module
10563        // GOODBYE silently too -- send it before propagating the error.
10564        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10565            Ok(routes) => routes,
10566            Err(err) => {
10567                warn!(
10568                    module_id = %spec.module_id,
10569                    ?reason,
10570                    error = %err,
10571                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10572                );
10573                send_module_goodbye(&spec.module_id, forwarding, target);
10574                return Err(SuperviseError::Forwarding(err));
10575            }
10576        };
10577        let route_goodbye_count = released_routes.len();
10578        send_route_goodbyes(forwarding, released_routes);
10579        send_module_goodbye(&spec.module_id, forwarding, target);
10580
10581        // The drain's happy path was previously silent: every emission above is
10582        // best-effort with only its failure arm logged, so "were consumers told"
10583        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10584        // hang where the open question was exactly whether teardown notice went
10585        // out). One summary line makes that class decidable in one grep.
10586        info!(
10587            module_id = %spec.module_id,
10588            ?reason,
10589            routes_notified,
10590            route_goodbyes = route_goodbye_count,
10591            abandoned_reservations = target.abandoned_bindings.len(),
10592            excluded_subscriptions = target.excluded_subscriptions,
10593            drained,
10594            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10595        );
10596    }
10597
10598    Ok(StopNotice::SentOverConnection)
10599}
10600
10601/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10602/// the only slot a plain (non-swap) spawn can register into.
10603async fn wait_for_registration_after_reload(
10604    registry: &Registry,
10605    module_id: &str,
10606    snapshot: &SharedSnapshot,
10607    child: &mut SupervisedChild,
10608    wait: Duration,
10609) -> Result<RegistrationWaitOutcome, SuperviseError> {
10610    wait_for_slot_registration(
10611        registry,
10612        crate::registry::RegistrationSlot::Active(module_id),
10613        module_id,
10614        snapshot,
10615        child,
10616        wait,
10617    )
10618    .await
10619}
10620
10621/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10622///
10623/// Keyed on the slot rather than the bare module id because during a swap the
10624/// id's active slot is already held by the incumbent: an id-keyed wait would
10625/// report the incumbent's registration as the candidate's and a candidate that
10626/// never registers would look registered. A swap candidate waits on
10627/// `crate::registry::RegistrationSlot::Candidate`.
10628async fn wait_for_slot_registration(
10629    registry: &Registry,
10630    slot: crate::registry::RegistrationSlot<'_>,
10631    module_id: &str,
10632    snapshot: &SharedSnapshot,
10633    child: &mut SupervisedChild,
10634    wait: Duration,
10635) -> Result<RegistrationWaitOutcome, SuperviseError> {
10636    let deadline = Instant::now() + wait;
10637    loop {
10638        if registry
10639            .registration(slot)
10640            .map_err(SuperviseError::Registry)?
10641            .is_some()
10642        {
10643            return Ok(RegistrationWaitOutcome::Registered);
10644        }
10645
10646        let now = Instant::now();
10647        if now >= deadline {
10648            return Ok(RegistrationWaitOutcome::TimedOut);
10649        }
10650        let remaining = deadline.saturating_duration_since(now);
10651        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10652
10653        tokio::select! {
10654            wait_result = child.wait() => {
10655                let status = wait_result.map_err(|source| SuperviseError::Wait {
10656                    module_id: module_id.to_string(),
10657                    source,
10658                })?;
10659                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10660                    snapshot,
10661                    child,
10662                    &status,
10663                )));
10664            }
10665            _ = sleep(poll) => {}
10666        }
10667    }
10668}
10669
10670fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10671    // A replacement process that exits before HELLO did not provide service, even
10672    // if it used status 0. Count it against the restart cap as a new-binary failure.
10673    if exit_report.kind != ExitKind::DeliberateSeverance {
10674        exit_report.kind = ExitKind::Crash;
10675    }
10676    exit_report
10677}
10678
10679async fn handle_reload_child_registration_failure(
10680    spec: &ModuleSpec,
10681    runtime: &SupervisorRuntimeConfig,
10682    registry: &Registry,
10683    process_liveness: &SupervisorProcessLiveness,
10684    snapshot: &SharedSnapshot,
10685    _child: &mut Option<SupervisedChild>,
10686    failure: ReloadRegistrationFailure,
10687) -> Result<(), SuperviseError> {
10688    let ReloadRegistrationFailure {
10689        exit_report,
10690        reason,
10691    } = failure;
10692    match on_child_exit(
10693        spec,
10694        runtime.restart_policy,
10695        registry,
10696        snapshot,
10697        &runtime.terminal_ring,
10698        &runtime.spawn_events,
10699        &runtime.child_roster,
10700        exit_report,
10701    )
10702    .await
10703    {
10704        NextAction::Stop {
10705            registration_released,
10706        } => {
10707            if registration_released {
10708                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10709            }
10710        }
10711        NextAction::Restart { schedule } => {
10712            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10713                schedule.delay
10714            });
10715            if let Some(schedule) = schedule {
10716                log_crash_respawn(&spec.module_id, schedule);
10717            }
10718            schedule_respawn(
10719                runtime,
10720                snapshot,
10721                &spec.module_id,
10722                delay,
10723                RespawnKind::Spawn,
10724            )?;
10725        }
10726    }
10727    Err(SuperviseError::ReloadFailed {
10728        module_id: spec.module_id.clone(),
10729        reason,
10730    })
10731}
10732
10733async fn handle_reload_spawn_failure(
10734    spec: &ModuleSpec,
10735    runtime: &SupervisorRuntimeConfig,
10736    process_liveness: &SupervisorProcessLiveness,
10737    snapshot: &SharedSnapshot,
10738    _child: &mut Option<SupervisedChild>,
10739    reason: String,
10740) -> Result<(), SuperviseError> {
10741    let now = Instant::now();
10742    let mut schedule = None;
10743    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10744        clear_current_process_facts(state);
10745        if state.enabled {
10746            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10747            state.state = if schedule.is_some() {
10748                ModuleState::Restarting
10749            } else {
10750                ModuleState::Failed
10751            };
10752        } else {
10753            state.state = ModuleState::Disabled;
10754        }
10755    })?;
10756    if let Some(schedule) = schedule {
10757        schedule_respawn(
10758            runtime,
10759            snapshot,
10760            &spec.module_id,
10761            schedule.delay,
10762            RespawnKind::Spawn,
10763        )?;
10764    } else {
10765        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10766    }
10767    Err(SuperviseError::ReloadFailed {
10768        module_id: spec.module_id.clone(),
10769        reason,
10770    })
10771}
10772
10773fn control_flags() -> Flags {
10774    Flags::new(false, Priority::Passive, false)
10775}
10776
10777#[allow(clippy::too_many_arguments)]
10778async fn drain_optional_child(
10779    module_id: &str,
10780    protocol: ModuleProtocol,
10781    stop_notice: StopNotice,
10782    registry: &Registry,
10783    forwarding: Option<&ForwardingTable>,
10784    snapshot: &SharedSnapshot,
10785    terminal_ring: &Arc<Mutex<TerminalRing>>,
10786    spawn_events: &SpawnEventFeed,
10787    child: &mut Option<SupervisedChild>,
10788    drain_timeout: Duration,
10789    final_state: ModuleState,
10790    enabled: Option<bool>,
10791) -> Result<(), SuperviseError> {
10792    if let Some(child) = child.take() {
10793        drain_child_to_state(
10794            module_id,
10795            protocol,
10796            stop_notice,
10797            registry,
10798            forwarding,
10799            snapshot,
10800            terminal_ring,
10801            spawn_events,
10802            child,
10803            drain_timeout,
10804            final_state,
10805            enabled,
10806        )
10807        .await
10808    } else {
10809        update_snapshot(snapshot, Some(module_id), |state| {
10810            state.state = final_state;
10811            if let Some(enabled) = enabled {
10812                state.enabled = enabled;
10813            }
10814            clear_current_process_facts(state);
10815        })?;
10816        release_dead_registration(registry, forwarding, snapshot, module_id).await
10817    }
10818}
10819
10820#[allow(clippy::too_many_arguments)]
10821async fn drain_child_to_state(
10822    module_id: &str,
10823    _protocol: ModuleProtocol,
10824    stop_notice: StopNotice,
10825    registry: &Registry,
10826    forwarding: Option<&ForwardingTable>,
10827    snapshot: &SharedSnapshot,
10828    terminal_ring: &Arc<Mutex<TerminalRing>>,
10829    spawn_events: &SpawnEventFeed,
10830    mut child: SupervisedChild,
10831    drain_timeout: Duration,
10832    final_state: ModuleState,
10833    enabled: Option<bool>,
10834) -> Result<(), SuperviseError> {
10835    let protocol = child.protocol;
10836    update_snapshot(snapshot, Some(module_id), |state| {
10837        state.state = ModuleState::Draining;
10838        state.draining_to_replace = final_state == ModuleState::Restarting;
10839        if let Some(enabled) = enabled {
10840            state.enabled = enabled;
10841        }
10842    })?;
10843
10844    // The wait below is the same budget in every case; what differs is
10845    // whether anything has ASKED the child to stop before it starts. Only a
10846    // forwarding drain that reached the module's registered connection has
10847    // (`module.draining`, then a module GOODBYE). Every other child was told
10848    // nothing: a `protocol: "none"` module, which never registers; a subc
10849    // module spawned moments ago that has not sent HELLO yet; or a stop that
10850    // runs no forwarding drain. Without a signal the budget is only a delay
10851    // in front of SIGKILL -- and the not-yet-registered child is the worst
10852    // case, because it registers into a module that is already draining,
10853    // is never told, and is killed while healthy.
10854    if stop_notice != StopNotice::SentOverConnection {
10855        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10856            info!(
10857                module_id,
10858                pid = child.pid,
10859                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10860                "module has no connection yet; requesting stop by signal"
10861            );
10862        }
10863        request_graceful_stop(module_id, &child);
10864    }
10865
10866    let exit_report = match timeout(drain_timeout, child.wait()).await {
10867        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10868        Ok(Err(source)) => {
10869            fail_after_wait_error(
10870                module_id,
10871                snapshot,
10872                terminal_ring,
10873                spawn_events,
10874                format!("failed to wait during drain: {source}"),
10875            );
10876            return Err(SuperviseError::Wait {
10877                module_id: module_id.to_string(),
10878                source,
10879            });
10880        }
10881        Err(_) => {
10882            // Mirror the sibling arm above: state is already `Draining`, and an
10883            // error propagated from here would strand it there -- a state
10884            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10885            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10886            // `Failed` before `?` keeps the module operator-visible and
10887            // revivable. Trigger is an ESRCH race (process exits between the
10888            // drain timeout firing and the kill) or a post-kill wait failure
10889            // (issue #34).
10890            //
10891            // Logged because the kill is otherwise visible only as signal 9 in
10892            // the terminal ring, and the budget it follows can be long enough
10893            // that consumers see a stretch of refusals with no stated cause.
10894            warn!(
10895                module_id,
10896                pid = child.pid,
10897                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10898                reason = ?final_state,
10899                ?stop_notice,
10900                "drain budget expired before the module exited; killing it"
10901            );
10902            child.start_kill().map_err(|source| {
10903                fail_after_wait_error(
10904                    module_id,
10905                    snapshot,
10906                    terminal_ring,
10907                    spawn_events,
10908                    format!("failed to kill during drain: {source}"),
10909                );
10910                SuperviseError::Kill {
10911                    module_id: module_id.to_string(),
10912                    source,
10913                }
10914            })?;
10915            let status = child.wait().await.map_err(|source| {
10916                fail_after_wait_error(
10917                    module_id,
10918                    snapshot,
10919                    terminal_ring,
10920                    spawn_events,
10921                    format!("failed to wait after drain kill: {source}"),
10922                );
10923                SuperviseError::Wait {
10924                    module_id: module_id.to_string(),
10925                    source,
10926                }
10927            })?;
10928            classify_reaped_child_exit(snapshot, &child, &status)
10929        }
10930    };
10931
10932    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10933    publish_after_terminal_record(
10934        module_id,
10935        || {
10936            record_terminal_with_detail(
10937                module_id,
10938                terminal_ring,
10939                spawn_events,
10940                &exit_report,
10941                terminal_disposition(final_state),
10942                detail,
10943            )
10944        },
10945        || {
10946            update_snapshot(snapshot, Some(module_id), |state| {
10947                state.state = final_state;
10948                if let Some(enabled) = enabled {
10949                    state.enabled = enabled;
10950                }
10951                clear_current_process_facts(state);
10952                state.last_exit = Some(exit_report.clone());
10953                if exit_report.kind == ExitKind::DeliberateSeverance {
10954                    state.lifetime_restarts += 1;
10955                }
10956            })
10957        },
10958    )?;
10959    child.drain_stderr(module_id).await;
10960
10961    release_dead_registration(registry, forwarding, snapshot, module_id).await
10962}
10963
10964/// Ask a child that nothing else has asked to stop, by signal.
10965///
10966/// A registered subc module is asked over its own connection: the drain sends
10967/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10968/// module GOODBYE, and the module stops itself. A module that speaks no subc
10969/// wire receives none of that, and neither does a subc module that has not
10970/// registered yet, so for them the drain budget would be pure delay in front of
10971/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10972/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10973/// into a recovery on the next start.
10974///
10975/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10976/// rule rather than an optimisation: that module's graceful stop is already
10977/// running by the time its child is drained, and a signal would race it.
10978///
10979/// Best-effort by construction. A child that has already exited is the ordinary
10980/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10981/// failure is logged at debug and the wait-then-kill below still decides the
10982/// outcome.
10983#[cfg(unix)]
10984fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10985    let Some(pid) = child
10986        .id()
10987        .and_then(|pid| i32::try_from(pid).ok())
10988        .and_then(rustix::process::Pid::from_raw)
10989    else {
10990        debug!(
10991            module_id,
10992            "no pid to signal for teardown; falling through to the drain wait"
10993        );
10994        return;
10995    };
10996    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10997        Ok(()) => debug!(
10998            module_id,
10999            "sent SIGTERM to a module nothing else asked to stop"
11000        ),
11001        Err(err) => debug!(
11002            module_id,
11003            error = %err,
11004            "SIGTERM to module failed; the drain wait and kill still apply"
11005        ),
11006    }
11007}
11008
11009/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11010/// Windows does offer need cooperation this supervisor cannot assume: a console
11011/// control event requires sharing a console with the child, and `WM_CLOSE`
11012/// requires the child to pump a message loop. A supervised server process does
11013/// neither, so there is nothing to send and teardown is the wait followed by the
11014/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11015/// the thing `protocol: "none"` exists to avoid.
11016#[cfg(not(unix))]
11017fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11018    debug!(
11019        module_id,
11020        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11021    );
11022}
11023
11024fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11025    match final_state {
11026        ModuleState::Stopped => TerminalDisposition::Stopped,
11027        ModuleState::Disabled => TerminalDisposition::Disabled,
11028        ModuleState::Restarting => TerminalDisposition::Restarting,
11029        ModuleState::Failed => TerminalDisposition::Failed,
11030        ModuleState::Starting
11031        | ModuleState::Running
11032        | ModuleState::Unresponsive
11033        | ModuleState::Draining => {
11034            unreachable!("terminal exits only finish in terminal or restarting states")
11035        }
11036    }
11037}
11038
11039/// Release a reaped child's registration before allowing another spawn.
11040///
11041/// EOF is not a process-lifetime signal: an inherited socket can stay open
11042/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11043/// reading EOF. After the normal release grace, request connection close (which
11044/// cancels both reads and dispatch), then allow one more release grace for the
11045/// connection guard's forwarding cleanup. Never evict a different connection.
11046async fn release_dead_registration(
11047    registry: &Registry,
11048    forwarding: Option<&ForwardingTable>,
11049    snapshot: &SharedSnapshot,
11050    module_id: &str,
11051) -> Result<(), SuperviseError> {
11052    let result = async {
11053        let registration = registry
11054            .get_module(module_id)
11055            .map_err(SuperviseError::Registry)?;
11056        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11057            Ok(()) => return Ok(()),
11058            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11059            Err(err) => return Err(err),
11060        }
11061        let pid = lock_snapshot(snapshot)?.reaped_pid;
11062        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11063            warn!(
11064                module_id,
11065                pid,
11066                connection_id = registration.connection_id.get(),
11067                "reaped module registration outlived release grace; closing dead connection"
11068            );
11069            forwarding.request_connection_close(
11070                registration.connection_id,
11071                CloseReason::new(
11072                    "supervised_process_reaped",
11073                    format!("module '{module_id}' pid {pid} exited"),
11074                ),
11075            );
11076            wait_for_slot_registration_release(
11077                registry,
11078                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11079                REGISTRY_RELEASE_TIMEOUT,
11080            )
11081            .await?;
11082        }
11083        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11084    }
11085    .await;
11086    if let Err(err) = &result {
11087        fail_snapshot(snapshot, Some(module_id), None);
11088        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11089    }
11090    result
11091}
11092
11093/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11094/// plain stop or restart waits for before it spawns a replacement.
11095async fn wait_for_registration_release(
11096    registry: &Registry,
11097    module_id: &str,
11098    wait: Duration,
11099) -> Result<(), SuperviseError> {
11100    wait_for_slot_registration_release(
11101        registry,
11102        crate::registry::RegistrationSlot::Active(module_id),
11103        wait,
11104    )
11105    .await
11106}
11107
11108/// Wait for the registration in `slot` to go away.
11109///
11110/// Keyed on the slot rather than the bare module id because a successful swap
11111/// never empties the id's active slot (the promoted candidate is in it), so an
11112/// id-keyed wait for the incumbent's release would always time out. Draining a
11113/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11114/// incumbent's connection instead.
11115async fn wait_for_slot_registration_release(
11116    registry: &Registry,
11117    slot: crate::registry::RegistrationSlot<'_>,
11118    wait: Duration,
11119) -> Result<(), SuperviseError> {
11120    let deadline = Instant::now() + wait;
11121    let mut release_events = registration_release_events().subscribe();
11122    let still_active = |registration: &crate::registry::ModuleRegistration| {
11123        SuperviseError::RegistrationStillActive {
11124            module_id: registration.manifest.module_id.clone(),
11125            waited: wait,
11126        }
11127    };
11128    loop {
11129        let _observed_generation = *release_events.borrow_and_update();
11130        let Some(registration) = registry
11131            .registration(slot)
11132            .map_err(SuperviseError::Registry)?
11133        else {
11134            return Ok(());
11135        };
11136
11137        let now = Instant::now();
11138        if now >= deadline {
11139            return Err(still_active(&registration));
11140        }
11141
11142        let remaining = deadline.saturating_duration_since(now);
11143        match timeout(remaining, release_events.changed()).await {
11144            Ok(Ok(())) | Ok(Err(_)) => {}
11145            Err(_) => return Err(still_active(&registration)),
11146        }
11147    }
11148}
11149
11150#[cfg(test)]
11151mod slot_registration_wait_tests {
11152    use super::*;
11153    use crate::registry::{ConnectionId, RegistrationSlot};
11154    use subc_protocol::manifest::ModuleManifest;
11155
11156    #[tokio::test]
11157    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11158        let registry = Arc::new(Registry::default());
11159        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11160        let runtime = supervisor.runtime_config();
11161        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11162        let spec = ModuleSpec {
11163            module_id: "enable-stale-registration".to_string(),
11164            program: PathBuf::from("/missing/enable-retry-test"),
11165            args: Vec::new(),
11166            env: Vec::new(),
11167            reserved: false,
11168            reserved_prefixes: Vec::new(),
11169            protocol: ModuleProtocol::Subc,
11170            overlap: Default::default(),
11171        };
11172        let connection = ConnectionId::new(90);
11173        registry
11174            .register_with_control_ops(
11175                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11176                1,
11177                connection,
11178                Vec::new(),
11179            )
11180            .unwrap();
11181        let mut child = None;
11182        let err = set_child_enabled(
11183            &spec,
11184            &runtime,
11185            &registry,
11186            &supervisor.process_liveness,
11187            &snapshot,
11188            &mut child,
11189            true,
11190        )
11191        .await
11192        .unwrap_err();
11193        assert!(matches!(
11194            err,
11195            SuperviseError::RegistrationStillActive { .. }
11196        ));
11197        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11198        assert!(child.is_none());
11199        registry.deregister_connection(connection).unwrap();
11200        let err = set_child_enabled(
11201            &spec,
11202            &runtime,
11203            &registry,
11204            &supervisor.process_liveness,
11205            &snapshot,
11206            &mut child,
11207            true,
11208        )
11209        .await
11210        .unwrap_err();
11211        assert!(
11212            matches!(err, SuperviseError::Spawn { .. }),
11213            "second enable must attempt a spawn: {err}"
11214        );
11215        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11216    }
11217
11218    const INCUMBENT: u64 = 1;
11219    const CANDIDATE: u64 = 2;
11220
11221    fn swapped_registry() -> Arc<Registry> {
11222        let registry = Arc::new(Registry::default());
11223        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11224        registry
11225            .register_with_control_ops(
11226                manifest.clone(),
11227                1,
11228                ConnectionId::new(INCUMBENT),
11229                Vec::new(),
11230            )
11231            .unwrap();
11232        registry
11233            .register_candidate_with_control_ops(
11234                manifest,
11235                1,
11236                ConnectionId::new(CANDIDATE),
11237                Vec::new(),
11238            )
11239            .unwrap();
11240        registry
11241    }
11242
11243    /// After a promotion the id's active slot is held by the new process, so an
11244    /// id-keyed wait for the incumbent's release can never succeed; the
11245    /// connection-keyed wait completes as soon as the incumbent deregisters.
11246    #[tokio::test]
11247    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11248        let registry = swapped_registry();
11249        registry.promote_candidate("m").unwrap().unwrap();
11250
11251        assert!(matches!(
11252            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11253            Err(SuperviseError::RegistrationStillActive { .. })
11254        ));
11255
11256        // Still held while the incumbent's connection has not deregistered.
11257        assert!(matches!(
11258            wait_for_slot_registration_release(
11259                &registry,
11260                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11261                Duration::from_millis(50),
11262            )
11263            .await,
11264            Err(SuperviseError::RegistrationStillActive { .. })
11265        ));
11266
11267        let releaser = Arc::clone(&registry);
11268        let release = tokio::spawn(async move {
11269            sleep(Duration::from_millis(20)).await;
11270            releaser
11271                .deregister_connection(ConnectionId::new(INCUMBENT))
11272                .unwrap();
11273            notify_registration_release();
11274        });
11275        wait_for_slot_registration_release(
11276            &registry,
11277            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11278            Duration::from_secs(5),
11279        )
11280        .await
11281        .expect("the incumbent's own registration is released");
11282        release.await.unwrap();
11283        assert!(registry.get_module("m").unwrap().is_some());
11284    }
11285
11286    /// The candidate slot is waited on separately from the active slot: the
11287    /// incumbent's registration neither holds up nor stands in for it.
11288    #[tokio::test]
11289    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11290        let registry = swapped_registry();
11291        assert!(matches!(
11292            wait_for_slot_registration_release(
11293                &registry,
11294                RegistrationSlot::Candidate("m"),
11295                Duration::from_millis(50),
11296            )
11297            .await,
11298            Err(SuperviseError::RegistrationStillActive { .. })
11299        ));
11300        registry
11301            .deregister_connection(ConnectionId::new(CANDIDATE))
11302            .unwrap();
11303        wait_for_slot_registration_release(
11304            &registry,
11305            RegistrationSlot::Candidate("m"),
11306            Duration::from_millis(50),
11307        )
11308        .await
11309        .expect("a candidate slot with no candidate is released");
11310        assert!(registry
11311            .registration(RegistrationSlot::Active("m"))
11312            .unwrap()
11313            .is_some());
11314    }
11315}
11316
11317fn classify_exit(status: &ExitStatus) -> ExitReport {
11318    ExitReport {
11319        kind: if status.success() {
11320            ExitKind::Clean
11321        } else {
11322            ExitKind::Crash
11323        },
11324        code: status.code(),
11325        signal: exit_signal(status),
11326        at_ms: unix_ms_now(),
11327    }
11328}
11329
11330/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11331/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11332/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11333/// disposition still must be `Failed` so the terminal ring is not silently missing
11334/// an entry, matching what `fail_snapshot` records for this same arm.
11335fn wait_error_exit_report() -> ExitReport {
11336    ExitReport {
11337        kind: ExitKind::Crash,
11338        code: None,
11339        signal: None,
11340        at_ms: unix_ms_now(),
11341    }
11342}
11343
11344#[cfg(unix)]
11345fn exit_signal(status: &ExitStatus) -> Option<i32> {
11346    use std::os::unix::process::ExitStatusExt;
11347
11348    status.signal()
11349}
11350
11351#[cfg(not(unix))]
11352fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11353    None
11354}
11355
11356/// Give an operator-touched module its full crash budget back.
11357///
11358/// Named for the counter it used to zero; it now empties the in-window ring,
11359/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11360/// ledger of what happened survives every operator action.
11361fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11362    update_snapshot(snapshot, Some(module_id), |state| {
11363        state.clear_crash_restarts();
11364    })
11365}
11366
11367fn set_running(
11368    snapshot: &SharedSnapshot,
11369    child: &SupervisedChild,
11370    module_id: &str,
11371    spawn_events: &SpawnEventFeed,
11372) -> Result<(), SuperviseError> {
11373    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11374        module_id: Some(module_id.to_string()),
11375    })?;
11376    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11377    if std::mem::take(&mut state.coalesced_restart_pending) {
11378        let generation = state.spawn_generation;
11379        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11380    }
11381    state.drain_disposition_detail = None;
11382    state.spawn_failure = None;
11383    state.waiting_for_previous_pid = None;
11384    state.waiting_for_previous_detail = None;
11385    // Every caller of this is a plain spawn, which always uses the primary key;
11386    // a promoted swap candidate sets the flag itself after this returns.
11387    state.in_alternate_slot = false;
11388    state.configuration_updated_since_spawn = false;
11389    state.spawned_protocol = Some(child.protocol);
11390    state.state = ModuleState::Running;
11391    state.enabled = true;
11392    state.process_alive = true;
11393    state.pid = child.id();
11394    // Probe samples belong to one process. Keep module-level escalation and
11395    // late-answer history, but never charge a new child for its predecessor's misses.
11396    state.health.status = SupervisorHealthStatus::Unknown;
11397    state.health.consecutive_failures = 0;
11398    state.health.detail = None;
11399    state.health.metrics = None;
11400    state.health.last_probe_ms = None;
11401    #[cfg(windows)]
11402    {
11403        state.pipe_consumed = child
11404            .nonce_delivery
11405            .as_ref()
11406            .map(|delivery| delivery.consumed());
11407    }
11408    #[cfg(target_os = "macos")]
11409    {
11410        state.report_ready = Some(Arc::clone(&child.report_ready));
11411    }
11412    state.spawned_at_ms = Some(child.spawned_at_ms);
11413    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11414    state.spawned_from = Some(child.spawned_from.clone());
11415    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11416    state.spawned_file_identity = child.spawned_file_identity.clone();
11417    state.process_start_time = child.process_start_time;
11418    Ok(())
11419}
11420
11421fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11422    state.waiting_for_previous_pid = None;
11423    state.waiting_for_previous_detail = None;
11424    state.process_alive = false;
11425    state.spawned_protocol = None;
11426    state.pid = None;
11427    #[cfg(windows)]
11428    {
11429        state.pipe_consumed = None;
11430    }
11431    #[cfg(target_os = "macos")]
11432    {
11433        state.report_ready = None;
11434    }
11435    state.spawned_at_ms = None;
11436    state.exec_outcome = None;
11437    state.spawned_from = None;
11438    state.launch_spec_digest = None;
11439    state.spawned_file_identity = None;
11440    state.process_start_time = None;
11441    state.deliberate_severance = None;
11442}
11443
11444#[cfg(test)]
11445fn record_deliberate_severance(
11446    snapshot: &SharedSnapshot,
11447    identity: ProcessIdentity,
11448) -> Result<(), SuperviseError> {
11449    update_snapshot(snapshot, None, |state| {
11450        state.deliberate_severance = Some(identity);
11451    })
11452}
11453
11454fn apply_deliberate_severance_marker(
11455    snapshot: &SharedSnapshot,
11456    exited_identity: Option<ProcessIdentity>,
11457    mut exit_report: ExitReport,
11458) -> ExitReport {
11459    let marker = lock_snapshot(snapshot)
11460        .ok()
11461        .and_then(|mut state| state.deliberate_severance.take());
11462    if marker.is_some() && marker == exited_identity {
11463        exit_report.kind = ExitKind::DeliberateSeverance;
11464    }
11465    exit_report
11466}
11467
11468fn classify_reaped_child_exit(
11469    snapshot: &SharedSnapshot,
11470    child: &SupervisedChild,
11471    status: &ExitStatus,
11472) -> ExitReport {
11473    let _ = update_snapshot(snapshot, None, |state| {
11474        state.reaped_pid = Some(child.pid);
11475        state.spawn_failure = child.spawn_failure.clone();
11476    });
11477    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11478}
11479
11480fn fail_snapshot(
11481    snapshot: &SharedSnapshot,
11482    module_id: Option<&str>,
11483    last_exit: Option<ExitReport>,
11484) {
11485    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11486        state.state = ModuleState::Failed;
11487        clear_current_process_facts(state);
11488        if let Some(last_exit) = last_exit {
11489            state.last_exit = Some(last_exit);
11490        }
11491    }) {
11492        error!(error = %err, "failed to mark supervisor state failed");
11493    }
11494}
11495
11496fn update_snapshot(
11497    snapshot: &SharedSnapshot,
11498    module_id: Option<&str>,
11499    update: impl FnOnce(&mut SupervisorSnapshot),
11500) -> Result<(), SuperviseError> {
11501    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11502        module_id: module_id.map(ToOwned::to_owned),
11503    })?;
11504    update(&mut state);
11505    Ok(())
11506}
11507
11508const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11509
11510fn lock_snapshot_for_control<'a>(
11511    snapshot: &'a SharedSnapshot,
11512    module_id: &str,
11513    caller: &'static str,
11514) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11515    let started_at = Instant::now();
11516    let guard = lock_snapshot(snapshot)?;
11517    let waited = started_at.elapsed();
11518    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11519        warn!(
11520            module_id = %module_id,
11521            waited_ms = waited.as_millis() as u64,
11522            caller = %caller,
11523            "slow snapshot lock"
11524        );
11525    }
11526    Ok(guard)
11527}
11528
11529fn lock_snapshot(
11530    snapshot: &SharedSnapshot,
11531) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11532    snapshot
11533        .lock()
11534        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11535}
11536
11537#[cfg(test)]
11538mod terminal_history_tests {
11539    use std::{
11540        path::PathBuf,
11541        sync::Arc,
11542        time::{Duration, Instant},
11543    };
11544
11545    use tokio::{sync::mpsc, time::sleep};
11546
11547    use super::{
11548        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11549        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11550        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11551        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11552        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11553        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11554        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11555        SupervisorHealthStatus, SupervisorSnapshot,
11556    };
11557    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11558    // use for their own wall-clock deadlines: crash-restart instants must be on
11559    // the same clock the production code stamps them with, which is tokio's (and
11560    // is what `start_paused` tests can move).
11561    use super::Instant as ClockInstant;
11562    use crate::{
11563        registry::Registry,
11564        terminal_ring::{TerminalRing, TerminalRingConfig},
11565    };
11566    use std::sync::Mutex;
11567    use subc_control::TerminalDisposition;
11568
11569    /// See the twin in `control.rs` for why this derives the path from
11570    /// `current_exe()` and why the existence check is here: `--lib` alone does
11571    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11572    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11573    pub(super) fn fake_aft_stub_path() -> PathBuf {
11574        let mut path = std::env::current_exe().expect("current_exe available in tests");
11575        path.pop();
11576        path.pop();
11577        path.push(if cfg!(windows) {
11578            "fake-aft-stub.exe"
11579        } else {
11580            "fake-aft-stub"
11581        });
11582        assert!(
11583            path.exists(),
11584            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11585             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11586            path.display()
11587        );
11588        path
11589    }
11590
11591    #[test]
11592    fn reserved_never_spawned_refuses_every_hello() {
11593        // The canary hole: a reserved id whose module has never spawned had NO
11594        // gate entry and admitted anyone -- the reservation protected the nonce
11595        // holder, not the NAME. Now the entry is present with no legitimate
11596        // holder and refuses all comers.
11597        let supervisor = SupervisorHandle::default();
11598        supervisor.apply_identity_configuration(&ModuleSpec {
11599            module_id: "never-spawned".to_string(),
11600            program: PathBuf::from("/usr/bin/false"),
11601            args: Vec::new(),
11602            env: Vec::new(),
11603            reserved: true,
11604            reserved_prefixes: Vec::new(),
11605            protocol: ModuleProtocol::Subc,
11606            overlap: Default::default(),
11607        });
11608        assert!(
11609            supervisor
11610                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11611                .is_err(),
11612            "forged nonce must refuse on a reserved never-spawned id"
11613        );
11614        assert!(
11615            supervisor
11616                .reserved_hello_verification("never-spawned", None)
11617                .is_err(),
11618            "absent nonce must refuse on a reserved never-spawned id"
11619        );
11620        // And a real spawn nonce minted later admits exactly that nonce.
11621        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11622        supervisor.apply_identity_configuration(&ModuleSpec {
11623            module_id: "never-spawned".to_string(),
11624            program: PathBuf::from("/usr/bin/false"),
11625            args: Vec::new(),
11626            env: Vec::new(),
11627            reserved: true,
11628            reserved_prefixes: Vec::new(),
11629            protocol: ModuleProtocol::Subc,
11630            overlap: Default::default(),
11631        });
11632        assert!(supervisor
11633            .reserved_hello_verification("never-spawned", Some("minted"))
11634            .is_ok());
11635        assert!(supervisor
11636            .reserved_hello_verification("never-spawned", Some("forged"))
11637            .is_err());
11638    }
11639
11640    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11641    /// happened, which is what "spent budget" looks like to every reader.
11642    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11643        let now = ClockInstant::now();
11644        for _ in 0..count {
11645            state.crash_restarts.push_back(now);
11646        }
11647    }
11648
11649    /// Age the oldest recorded restart out of `window`, standing in for the hours
11650    /// that would otherwise have to pass. Injecting the instant is the point: a
11651    /// test that slept a real window would take ten minutes and still prove less.
11652    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11653        let aged = state
11654            .crash_restarts
11655            .front()
11656            .expect("a crash restart must be recorded before it can be aged")
11657            .checked_sub(window + Duration::from_secs(1))
11658            .expect("the test clock is far enough from its origin to age an instant");
11659        state.crash_restarts[0] = aged;
11660    }
11661
11662    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11663        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11664        seed_crash_restarts(&mut state, count);
11665        state
11666    }
11667
11668    #[test]
11669    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11670        let policy = RestartPolicy::new(3, Duration::ZERO);
11671        let now = ClockInstant::now();
11672        assert!(daemon_will_restart(
11673            &mut snapshot_with_restarts(true, 2),
11674            &policy,
11675            now
11676        ));
11677        assert!(!daemon_will_restart(
11678            &mut snapshot_with_restarts(true, 3),
11679            &policy,
11680            now
11681        ));
11682        assert!(!daemon_will_restart(
11683            &mut snapshot_with_restarts(false, 0),
11684            &policy,
11685            now
11686        ));
11687    }
11688
11689    #[test]
11690    fn crash_restart_backoff_escalates_with_in_window_count() {
11691        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11692            .with_max_backoff(Duration::from_secs(30));
11693        let now = ClockInstant::now();
11694        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11695        let schedules = (0..4)
11696            .map(|_| {
11697                state
11698                    .next_crash_restart(&policy, now)
11699                    .expect("the test policy allows four crash restarts")
11700            })
11701            .collect::<Vec<_>>();
11702
11703        assert_eq!(
11704            schedules
11705                .iter()
11706                .map(|schedule| schedule.restart_in_window)
11707                .collect::<Vec<_>>(),
11708            vec![0, 1, 2, 3]
11709        );
11710        assert_eq!(
11711            schedules
11712                .iter()
11713                .map(|schedule| schedule.delay)
11714                .collect::<Vec<_>>(),
11715            vec![
11716                Duration::from_millis(100),
11717                Duration::from_secs(1),
11718                Duration::from_secs(10),
11719                Duration::from_secs(30),
11720            ]
11721        );
11722    }
11723
11724    #[test]
11725    fn crash_restart_backoff_resets_after_ring_clear() {
11726        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11727        let now = ClockInstant::now();
11728        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11729        assert_eq!(
11730            state.next_crash_restart(&policy, now).unwrap().delay,
11731            Duration::from_millis(100)
11732        );
11733        assert_eq!(
11734            state.next_crash_restart(&policy, now).unwrap().delay,
11735            Duration::from_secs(1)
11736        );
11737
11738        state.clear_crash_restarts();
11739        let schedule = state
11740            .next_crash_restart(&policy, now)
11741            .expect("a cleared ring must allow another restart");
11742        assert_eq!(schedule.restart_in_window, 0);
11743        assert_eq!(schedule.delay, Duration::from_millis(100));
11744    }
11745
11746    #[test]
11747    fn crash_restart_backoff_ignores_aged_restarts() {
11748        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11749        let now = ClockInstant::now();
11750        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11751        state
11752            .next_crash_restart(&policy, now)
11753            .expect("the first restart is allowed");
11754        state
11755            .next_crash_restart(&policy, now)
11756            .expect("the second restart is allowed");
11757        state.crash_restarts[0] = now
11758            .checked_sub(policy.window + Duration::from_secs(1))
11759            .expect("the fake clock can age a restart past the window");
11760
11761        let schedule = state
11762            .next_crash_restart(&policy, now)
11763            .expect("an aged restart must release its slot");
11764        assert_eq!(schedule.restart_in_window, 1);
11765        assert_eq!(schedule.delay, Duration::from_secs(1));
11766        assert_eq!(state.crash_restarts.len(), 2);
11767    }
11768
11769    /// The budget is a rate: the same three spent restarts refuse a respawn
11770    /// while they are recent and allow one once they have aged past the window.
11771    /// Nothing about the module changed in between, which is the whole point.
11772    #[test]
11773    fn a_budget_spent_before_the_window_no_longer_refuses() {
11774        let policy = RestartPolicy::new(3, Duration::ZERO);
11775        let mut state = snapshot_with_restarts(true, 3);
11776        let now = ClockInstant::now();
11777        assert!(!daemon_will_restart(&mut state, &policy, now));
11778
11779        assert!(daemon_will_restart(
11780            &mut state,
11781            &policy,
11782            now + policy.window + Duration::from_secs(1)
11783        ));
11784        assert!(
11785            state.crash_restarts.is_empty(),
11786            "reading the budget must drop the instants that left the window"
11787        );
11788    }
11789
11790    fn module_with_recovery_snapshot(
11791        state: ModuleState,
11792        enabled: bool,
11793        restart_count: u32,
11794    ) -> SupervisedModule {
11795        let registry = Arc::new(Registry::default());
11796        let supervisor =
11797            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11798        let runtime = supervisor.runtime_config();
11799        let spec = ModuleSpec {
11800            module_id: "recovery-snapshot".to_string(),
11801            program: fake_aft_stub_path(),
11802            args: Vec::new(),
11803            env: Vec::new(),
11804            reserved: false,
11805            reserved_prefixes: Vec::new(),
11806            protocol: ModuleProtocol::Subc,
11807            overlap: Default::default(),
11808        };
11809        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11810        seed_crash_restarts(&mut snapshot, restart_count);
11811        // These tests read synthetic snapshots. A real child and monitor would
11812        // race those reads by replacing the requested state during startup.
11813        let (commands, _rx) = mpsc::channel(4);
11814        SupervisedModule {
11815            inner: Arc::new(SupervisedModuleInner {
11816                module_id: spec.module_id.clone(),
11817                registry,
11818                snapshot: Arc::new(Mutex::new(snapshot)),
11819                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11820                    spec,
11821                    health: runtime.health,
11822                })),
11823                stderr_ring: runtime.stderr_ring,
11824                terminal_ring: runtime.terminal_ring,
11825                commands,
11826                monitor: Mutex::new(None),
11827                restart_policy: runtime.restart_policy,
11828                effective_drain_timeout: runtime.effective_drain_timeout,
11829                provenance_probe: supervisor.provenance_probe.clone(),
11830            }),
11831        }
11832    }
11833
11834    #[cfg(target_os = "linux")]
11835    #[tokio::test]
11836    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11837        let supervisor =
11838            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11839                .with_cgroup_placement(None);
11840        let result = supervisor.spawn(ModuleSpec {
11841            module_id: "no-cgroup-placement".to_string(),
11842            program: fake_aft_stub_path(),
11843            args: Vec::new(),
11844            env: Vec::new(),
11845            reserved: false,
11846            reserved_prefixes: Vec::new(),
11847            protocol: ModuleProtocol::Subc,
11848            overlap: Default::default(),
11849        });
11850
11851        assert!(
11852            result.is_ok(),
11853            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11854        );
11855    }
11856
11857    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11858    async fn undecided_snapshot_uses_shared_restart_predicate() {
11859        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11860            .will_recover_after_connection_loss()
11861            .unwrap());
11862        assert!(
11863            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11864                .will_recover_after_connection_loss()
11865                .unwrap()
11866        );
11867    }
11868
11869    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11870    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11871        assert!(
11872            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11873                .will_recover_after_connection_loss()
11874                .unwrap()
11875        );
11876    }
11877
11878    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11879    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11880        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11881            .will_recover_after_connection_loss()
11882            .unwrap());
11883        assert!(
11884            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11885                .will_recover_after_connection_loss()
11886                .unwrap()
11887        );
11888    }
11889
11890    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11891    async fn warming_snapshot_is_limited_to_startup_phases() {
11892        for state in [
11893            ModuleState::Starting,
11894            ModuleState::Running,
11895            ModuleState::Restarting,
11896        ] {
11897            assert!(
11898                module_with_recovery_snapshot(state, true, 0)
11899                    .is_warming()
11900                    .unwrap(),
11901                "{state:?} should be warming"
11902            );
11903        }
11904        for state in [
11905            ModuleState::Unresponsive,
11906            ModuleState::Draining,
11907            ModuleState::Stopped,
11908            ModuleState::Failed,
11909            ModuleState::Disabled,
11910        ] {
11911            assert!(
11912                !module_with_recovery_snapshot(state, true, 0)
11913                    .is_warming()
11914                    .unwrap(),
11915                "{state:?} should not be warming"
11916            );
11917        }
11918    }
11919
11920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11921    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11922        let registry = Arc::new(Registry::default());
11923        let supervisor =
11924            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11925        let module = supervisor
11926            .spawn(ModuleSpec {
11927                module_id: "terminal-history".to_string(),
11928                program: fake_aft_stub_path(),
11929                args: Vec::new(),
11930                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11931                reserved: false,
11932                reserved_prefixes: Vec::new(),
11933                protocol: ModuleProtocol::Subc,
11934                overlap: Default::default(),
11935            })
11936            .unwrap();
11937
11938        let deadline = Instant::now() + Duration::from_secs(5);
11939        loop {
11940            let history = module.terminal_history();
11941            if history.entries.len() == 2 {
11942                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11943                assert_eq!(history.dropped, 0);
11944                assert_eq!(
11945                    history
11946                        .entries
11947                        .iter()
11948                        .map(|entry| entry.exit_code)
11949                        .collect::<Vec<_>>(),
11950                    vec![Some(23), Some(23)]
11951                );
11952                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11953                return;
11954            }
11955            assert!(
11956                Instant::now() < deadline,
11957                "module did not retain two terminal exits: {history:?}"
11958            );
11959            sleep(Duration::from_millis(10)).await;
11960        }
11961    }
11962
11963    /// A disable issued while a crash respawn is still backing off must preempt
11964    /// that respawn: the operator's stop wins, the disable must not queue behind
11965    /// the backoff, and the module must never come back up afterwards.
11966    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11967    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11968        let backoff = Duration::from_secs(2);
11969        let supervisor = Supervisor::new_for_test(
11970            Arc::new(Registry::default()),
11971            RestartPolicy::new(10, backoff),
11972        );
11973        let module = supervisor
11974            .spawn(ModuleSpec {
11975                module_id: "disable-during-backoff".to_string(),
11976                program: fake_aft_stub_path(),
11977                args: Vec::new(),
11978                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11979                reserved: false,
11980                reserved_prefixes: Vec::new(),
11981                protocol: ModuleProtocol::Subc,
11982                overlap: Default::default(),
11983            })
11984            .unwrap();
11985
11986        // Wait for the first crash to put the module into its backoff window.
11987        let deadline = Instant::now() + Duration::from_secs(5);
11988        loop {
11989            if module.status().unwrap().state == ModuleState::Restarting {
11990                break;
11991            }
11992            assert!(
11993                Instant::now() < deadline,
11994                "module never entered the crash backoff"
11995            );
11996            sleep(Duration::from_millis(10)).await;
11997        }
11998
11999        let started = Instant::now();
12000        module.set_enabled(false).await.unwrap();
12001        let waited = started.elapsed();
12002
12003        assert!(
12004            waited < backoff / 2,
12005            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12006        );
12007        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12008
12009        // Outlast the backoff: the respawn it was counting down to must never run.
12010        sleep(backoff + Duration::from_millis(500)).await;
12011        let status = module.status().unwrap();
12012        assert_eq!(status.state, ModuleState::Disabled);
12013        assert_eq!(
12014            status.spawn_generation, 1,
12015            "module respawned after the operator disabled it"
12016        );
12017    }
12018
12019    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12020    /// the shape of nats-server, the program this rule exists for.
12021    #[cfg(unix)]
12022    fn protocol_none_sigterm_exits_clean_spec(
12023        module_id: &str,
12024        dir: &std::path::Path,
12025    ) -> (ModuleSpec, PathBuf, PathBuf) {
12026        let ready = dir.join("ready");
12027        let marker = dir.join("sigterm");
12028        let spec = ModuleSpec {
12029            module_id: module_id.to_string(),
12030            program: fake_aft_stub_path(),
12031            args: Vec::new(),
12032            env: vec![
12033                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12034                (
12035                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12036                    marker.display().to_string(),
12037                ),
12038                (
12039                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12040                    ready.display().to_string(),
12041                ),
12042            ],
12043            reserved: false,
12044            reserved_prefixes: Vec::new(),
12045            protocol: ModuleProtocol::None,
12046            overlap: Default::default(),
12047        };
12048        (spec, ready, marker)
12049    }
12050
12051    /// Wait for a file the child writes, so a signal is never sent before the
12052    /// child's SIGTERM handler is installed (the default disposition would
12053    /// kill it by signal and the exit would not be clean).
12054    #[cfg(unix)]
12055    async fn wait_for_file(path: &std::path::Path) {
12056        let deadline = Instant::now() + Duration::from_secs(10);
12057        while !path.exists() {
12058            assert!(
12059                Instant::now() < deadline,
12060                "{} never appeared",
12061                path.display()
12062            );
12063            sleep(Duration::from_millis(10)).await;
12064        }
12065    }
12066
12067    /// A protocol-none module that exits 0 because something OUTSIDE the
12068    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12069    /// the crash-path disposition rather than `stopped`.
12070    #[cfg(unix)]
12071    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12072    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12073        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12074        let (spec, ready, marker) =
12075            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12076        let supervisor = Supervisor::new_for_test(
12077            Arc::new(Registry::default()),
12078            RestartPolicy::new(3, Duration::ZERO),
12079        );
12080        let module = supervisor.spawn(spec).unwrap();
12081        wait_for_file(&ready).await;
12082        // The ready file proves the child installed its SIGTERM handler, not
12083        // that the supervisor has processed the privacy trampoline's exec
12084        // acknowledgement. On macOS status withholds the pid until then.
12085        let deadline = Instant::now() + Duration::from_secs(10);
12086        let first_pid = loop {
12087            let status = module.status().unwrap();
12088            if status.state == ModuleState::Running {
12089                if let Some(pid) = status.pid {
12090                    break pid;
12091                }
12092            }
12093            assert!(
12094                Instant::now() < deadline,
12095                "a running module must report its pid after exec confirmation: {status:?}"
12096            );
12097            sleep(Duration::from_millis(10)).await;
12098        };
12099
12100        rustix::process::kill_process(
12101            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12102            rustix::process::Signal::TERM,
12103        )
12104        .unwrap();
12105
12106        let deadline = Instant::now() + Duration::from_secs(10);
12107        let respawned = loop {
12108            let status = module.status().unwrap();
12109            if status.state == ModuleState::Running
12110                && status.pid.is_some_and(|pid| pid != first_pid)
12111            {
12112                break status;
12113            }
12114            assert!(
12115                Instant::now() < deadline,
12116                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12117            );
12118            sleep(Duration::from_millis(10)).await;
12119        };
12120        assert_eq!(respawned.spawn_generation, 2);
12121        assert!(
12122            marker.exists(),
12123            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12124        );
12125
12126        let history = module.terminal_history();
12127        assert_eq!(history.entries.len(), 1, "{history:?}");
12128        let entry = &history.entries[0];
12129        assert_eq!(entry.exit_code, Some(0));
12130        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12131        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12132
12133        module.stop().await.unwrap();
12134    }
12135
12136    /// Repeated unrequested clean exits of a protocol-none module spend the
12137    /// restart budget exactly as crashes do, and the module ends `failed` with
12138    /// the budget named.
12139    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12140    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12141        let supervisor = Supervisor::new_for_test(
12142            Arc::new(Registry::default()),
12143            RestartPolicy::new(1, Duration::ZERO),
12144        );
12145        let module = supervisor
12146            .spawn(ModuleSpec {
12147                module_id: "none-clean-exit-budget".to_string(),
12148                program: fake_aft_stub_path(),
12149                args: Vec::new(),
12150                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12151                reserved: false,
12152                reserved_prefixes: Vec::new(),
12153                protocol: ModuleProtocol::None,
12154                overlap: Default::default(),
12155            })
12156            .unwrap();
12157
12158        // Failed follows the terminal write; this deadline only bounds a hang,
12159        // not an assumed duration for the two launches or their exit recording.
12160        let deadline = Instant::now() + Duration::from_secs(10);
12161        loop {
12162            let status = module.status().unwrap();
12163            if status.state == ModuleState::Failed {
12164                break;
12165            }
12166            assert!(
12167                Instant::now() < deadline,
12168                "module never exhausted its budget: {status:?} {:?}",
12169                module.terminal_history()
12170            );
12171            sleep(Duration::from_millis(10)).await;
12172        }
12173        let history = module.terminal_history();
12174        assert_eq!(
12175            history
12176                .entries
12177                .iter()
12178                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12179                .collect::<Vec<_>>(),
12180            vec![
12181                (Some(0), TerminalDisposition::Restarting),
12182                (Some(0), TerminalDisposition::Failed),
12183            ]
12184        );
12185        let detail = history.entries[1]
12186            .disposition_detail
12187            .as_deref()
12188            .expect("a budget failure names the budget");
12189        assert!(detail.contains("max_restarts=1"), "{detail}");
12190        assert_eq!(module.status().unwrap().spawn_generation, 2);
12191    }
12192
12193    #[test]
12194    fn restart_budget_failure_is_published_after_its_terminal_record() {
12195        use std::io::{Read, Write};
12196        let supervisor = Supervisor::new_for_test(
12197            Arc::new(Registry::default()),
12198            RestartPolicy::new(0, Duration::ZERO),
12199        );
12200        let runtime = supervisor.runtime_config();
12201        let spec = ModuleSpec {
12202            protocol: ModuleProtocol::None,
12203            ..windowed_crash_spec("budget-publication-order")
12204        };
12205        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12206            ModuleState::Running,
12207            true,
12208        )));
12209        lock_snapshot(&snapshot).unwrap().process_alive = true;
12210        let reaped_snapshot = snapshot.clone();
12211        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12212        let history = runtime.terminal_ring.clone();
12213        let reap = std::thread::spawn(move || {
12214            tokio::runtime::Builder::new_current_thread()
12215                .enable_all()
12216                .build()
12217                .unwrap()
12218                .block_on(on_child_exit(
12219                    &spec,
12220                    runtime.restart_policy,
12221                    &supervisor.registry,
12222                    &reaped_snapshot,
12223                    &runtime.terminal_ring,
12224                    &runtime.spawn_events,
12225                    &runtime.child_roster,
12226                    ExitReport {
12227                        kind: ExitKind::Clean,
12228                        code: Some(0),
12229                        signal: None,
12230                        at_ms: 1,
12231                    },
12232                ))
12233        });
12234        let mut reached = [0];
12235        barrier.read_exact(&mut reached).unwrap();
12236        assert_eq!(&reached, b"R");
12237        let (before_state, before_alive) = {
12238            let state = lock_snapshot(&snapshot).unwrap();
12239            (state.state, state.process_alive)
12240        };
12241        let before_history = history.lock().unwrap().snapshot();
12242        barrier.write_all(b"X").unwrap();
12243        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12244        assert!(before_history.entries.is_empty());
12245        assert_ne!(
12246            before_state,
12247            ModuleState::Failed,
12248            "Failed was visible before its terminal record could be written"
12249        );
12250        assert!(before_alive);
12251        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12252        let history = history.lock().unwrap().snapshot();
12253        assert_eq!(history.entries.len(), 1);
12254        assert_eq!(history.entries[0].exit_code, Some(0));
12255        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12256    }
12257
12258    #[test]
12259    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12260        use std::io::{Read, Write};
12261
12262        let supervisor = Supervisor::new_for_test(
12263            Arc::new(Registry::default()),
12264            RestartPolicy::new(3, Duration::ZERO),
12265        );
12266        let mut runtime = supervisor.runtime_config();
12267        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12268        let snapshot = module.inner.snapshot.clone();
12269        lock_snapshot(&snapshot).unwrap().process_alive = true;
12270        runtime.terminal_ring = module.inner.terminal_ring.clone();
12271        let spec = windowed_crash_spec(module.module_id());
12272        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12273        let reaped_snapshot = snapshot.clone();
12274        let reap = std::thread::spawn(move || {
12275            tokio::runtime::Builder::new_current_thread()
12276                .enable_all()
12277                .build()
12278                .unwrap()
12279                .block_on(on_child_exit(
12280                    &spec,
12281                    runtime.restart_policy,
12282                    &supervisor.registry,
12283                    &reaped_snapshot,
12284                    &runtime.terminal_ring,
12285                    &runtime.spawn_events,
12286                    &runtime.child_roster,
12287                    ExitReport {
12288                        kind: ExitKind::Crash,
12289                        code: Some(2),
12290                        signal: None,
12291                        at_ms: 1,
12292                    },
12293                ))
12294        });
12295
12296        // Hold the crash handler between its budget decision and publication.
12297        // Connection cleanup must still see the recovery that this crash chose,
12298        // even when that recovery consumes the last available restart slot.
12299        let mut reached = [0];
12300        barrier.read_exact(&mut reached).unwrap();
12301        assert_eq!(&reached, b"R");
12302        let before_status = module.status().unwrap();
12303        let before_history = module.terminal_history();
12304        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12305        barrier.write_all(b"X").unwrap();
12306        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12307
12308        assert_eq!(before_status.state, ModuleState::Running);
12309        assert!(before_status.process_alive);
12310        assert_eq!(before_status.restart_count, 3);
12311        assert!(before_history.entries.is_empty());
12312        assert!(
12313            before_recovers,
12314            "connection cleanup would report a terminal close for a scheduled crash restart"
12315        );
12316        let restarting = module.status().unwrap();
12317        assert_eq!(restarting.state, ModuleState::Restarting);
12318        assert_eq!(restarting.restart_count, 3);
12319        assert!(module.will_recover_after_connection_loss().unwrap());
12320        let history = module.terminal_history();
12321        assert_eq!(history.entries.len(), 1);
12322        assert_eq!(
12323            history.entries[0].disposition,
12324            TerminalDisposition::Restarting
12325        );
12326
12327        // Once the replacement is running, its next crash must be judged
12328        // against the exhausted budget rather than the previous recovery.
12329        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12330            state.state = ModuleState::Running;
12331            state.respawn_pending = false;
12332        })
12333        .unwrap();
12334        assert!(!module.will_recover_after_connection_loss().unwrap());
12335        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12336            state.enabled = false;
12337            state.respawn_pending = true;
12338        })
12339        .unwrap();
12340        assert!(!module.will_recover_after_connection_loss().unwrap());
12341    }
12342
12343    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12344        use std::net::{TcpListener, TcpStream};
12345        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12346        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12347        let (barrier, _) = listener.accept().unwrap();
12348        for stream in [&writer, &barrier] {
12349            stream
12350                .set_read_timeout(Some(Duration::from_secs(10)))
12351                .unwrap();
12352            stream
12353                .set_write_timeout(Some(Duration::from_secs(10)))
12354                .unwrap();
12355        }
12356        *events.1.lock().unwrap() = Some(writer);
12357        barrier
12358    }
12359
12360    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12361        use std::io::{Read, Write};
12362        let barrier = events.1.lock().unwrap().take();
12363        if let Some(mut barrier) = barrier {
12364            barrier.write_all(b"R").unwrap();
12365            let mut release = [0];
12366            barrier.read_exact(&mut release).unwrap();
12367            assert_eq!(&release, b"X");
12368        }
12369    }
12370
12371    #[test]
12372    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12373        use std::io::{Read, Write};
12374
12375        let supervisor = Supervisor::new_for_test(
12376            Arc::new(Registry::default()),
12377            RestartPolicy::new(0, Duration::ZERO),
12378        );
12379        let mut runtime = supervisor.runtime_config();
12380        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12381        let snapshot = module.inner.snapshot.clone();
12382        runtime.terminal_ring = module.inner.terminal_ring.clone();
12383        let spec = ModuleSpec {
12384            module_id: module.module_id().into(),
12385            program: fake_aft_stub_path(),
12386            protocol: ModuleProtocol::None,
12387            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12388            ..windowed_crash_spec(module.module_id())
12389        };
12390        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12391        let reap = std::thread::spawn(move || {
12392            tokio::runtime::Builder::new_current_thread()
12393                .enable_all()
12394                .build()
12395                .unwrap()
12396                .block_on(async {
12397                    let mut child =
12398                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12399                    health_restart_child(
12400                        &spec,
12401                        &runtime,
12402                        &supervisor.registry,
12403                        &supervisor.process_liveness,
12404                        &snapshot,
12405                        &mut child,
12406                        SupervisorHealthStatus::Failing,
12407                        Some("health probe did not answer"),
12408                        1,
12409                    )
12410                    .await
12411                    .unwrap();
12412                    assert!(child.is_none());
12413                });
12414        });
12415        // The loopback barrier acknowledges entry into terminal recording by
12416        // health_restart_child's drain, then waits for the test's release byte.
12417        // Status and history are observed while that writer is held, not after
12418        // an elapsed-time guess about when the health action has completed.
12419        let mut reached = [0];
12420        barrier.read_exact(&mut reached).unwrap();
12421        assert_eq!(&reached, b"R");
12422        let before_status = module.status().unwrap();
12423        let before_history = module.terminal_history();
12424        barrier.write_all(b"X").unwrap();
12425        reap.join().unwrap();
12426        assert!(before_history.entries.is_empty());
12427        assert_ne!(
12428            before_status.state,
12429            ModuleState::Failed,
12430            "Failed was visible before its terminal record could be written"
12431        );
12432        assert!(
12433            before_status.process_alive,
12434            "process death was visible before its terminal record could be written"
12435        );
12436        let failed = module.status().unwrap();
12437        assert_eq!(failed.state, ModuleState::Failed);
12438        assert!(!failed.process_alive);
12439        let history = module.terminal_history();
12440        assert_eq!(history.entries.len(), 1);
12441        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12442        assert_eq!(
12443            history.entries[0].disposition_detail.as_deref(),
12444            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12445        );
12446    }
12447
12448    #[test]
12449    fn observed_exits_publish_process_death_after_terminal_history() {
12450        use std::io::{Read, Write};
12451        for (kind, enabled, shutdown, wait_error, expected_state) in [
12452            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12453            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12454            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12455            (
12456                ExitKind::DeliberateSeverance,
12457                true,
12458                false,
12459                false,
12460                ModuleState::Restarting,
12461            ),
12462            #[cfg(unix)]
12463            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12464            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12465        ] {
12466            let supervisor = Supervisor::new_for_test(
12467                Arc::new(Registry::default()),
12468                RestartPolicy::new(1, Duration::ZERO),
12469            );
12470            let runtime = supervisor.runtime_config();
12471            #[cfg(unix)]
12472            if shutdown {
12473                runtime.child_roster.close();
12474            }
12475            #[cfg(not(unix))]
12476            let _ = shutdown;
12477            let spec = windowed_crash_spec("exit-publication-order");
12478            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12479                ModuleState::Running,
12480                enabled,
12481            )));
12482            lock_snapshot(&snapshot).unwrap().process_alive = true;
12483            let reaped_snapshot = snapshot.clone();
12484            let history = runtime.terminal_ring.clone();
12485            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12486            let reap = std::thread::spawn(move || {
12487                if wait_error {
12488                    super::fail_after_wait_error(
12489                        &spec.module_id,
12490                        &reaped_snapshot,
12491                        &runtime.terminal_ring,
12492                        &runtime.spawn_events,
12493                        "wait failed".into(),
12494                    );
12495                } else {
12496                    tokio::runtime::Builder::new_current_thread()
12497                        .enable_all()
12498                        .build()
12499                        .unwrap()
12500                        .block_on(on_child_exit(
12501                            &spec,
12502                            runtime.restart_policy,
12503                            &supervisor.registry,
12504                            &reaped_snapshot,
12505                            &runtime.terminal_ring,
12506                            &runtime.spawn_events,
12507                            &runtime.child_roster,
12508                            ExitReport {
12509                                kind,
12510                                code: Some(0),
12511                                signal: None,
12512                                at_ms: 1,
12513                            },
12514                        ));
12515                }
12516            });
12517            let mut reached = [0];
12518            barrier.read_exact(&mut reached).unwrap();
12519            assert_eq!(&reached, b"R");
12520            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12521            let before_history = history.lock().unwrap().snapshot();
12522            barrier.write_all(b"X").unwrap();
12523            reap.join().unwrap();
12524            assert!(
12525                before_alive,
12526                "{kind:?}: death was published before recording"
12527            );
12528            assert!(before_history.entries.is_empty());
12529            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12530            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12531            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12532        }
12533    }
12534
12535    #[tokio::test]
12536    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12537        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12538        let path = dir.join("terminals.jsonl");
12539        std::fs::create_dir(&path).unwrap();
12540        let supervisor = Supervisor::new_for_test(
12541            Arc::new(Registry::default()),
12542            RestartPolicy::new(0, Duration::ZERO),
12543        )
12544        .with_terminal_journal(path, "health-budget-journal-failure".into());
12545        let runtime = supervisor.runtime_config();
12546        let spec = ModuleSpec {
12547            program: fake_aft_stub_path(),
12548            protocol: ModuleProtocol::None,
12549            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12550            ..windowed_crash_spec("health-budget-journal-failure")
12551        };
12552        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12553        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12554        health_restart_child(
12555            &spec,
12556            &runtime,
12557            &supervisor.registry,
12558            &supervisor.process_liveness,
12559            &snapshot,
12560            &mut child,
12561            SupervisorHealthStatus::Failing,
12562            None,
12563            1,
12564        )
12565        .await
12566        .unwrap();
12567        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12568        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12569        let history = runtime
12570            .terminal_ring
12571            .lock()
12572            .unwrap()
12573            .durable_history(&spec.module_id);
12574        assert!(history.journal_write_failures > 0);
12575        assert_eq!(history.entries.len(), 1);
12576        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12577    }
12578
12579    #[test]
12580    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12581        let supervisor = Supervisor::new_for_test(
12582            Arc::new(Registry::default()),
12583            RestartPolicy::new(0, Duration::ZERO),
12584        );
12585        let runtime = supervisor.runtime_config();
12586        let spec = ModuleSpec {
12587            program: fake_aft_stub_path(),
12588            protocol: ModuleProtocol::None,
12589            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12590            ..windowed_crash_spec("health-budget-recording-panic")
12591        };
12592        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12593        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12594            tokio::runtime::Builder::new_current_thread()
12595                .enable_all()
12596                .build()
12597                .unwrap()
12598                .block_on(async {
12599                    let mut child =
12600                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12601                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12602                    health_restart_child(
12603                        &spec,
12604                        &runtime,
12605                        &supervisor.registry,
12606                        &supervisor.process_liveness,
12607                        &snapshot,
12608                        &mut child,
12609                        SupervisorHealthStatus::Failing,
12610                        None,
12611                        1,
12612                    )
12613                    .await
12614                    .unwrap();
12615                });
12616        }));
12617        let panic = result.expect_err("recording must still unwind");
12618        assert_eq!(
12619            panic.downcast_ref::<String>().map(String::as_str),
12620            Some("spawn event sequence exhausted")
12621        );
12622        let state = lock_snapshot(&snapshot).unwrap();
12623        assert_eq!(state.state, ModuleState::Failed);
12624        assert!(!state.process_alive);
12625        assert!(state.last_exit.is_some());
12626    }
12627
12628    #[tokio::test]
12629    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12630        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12631        let path = dir.join("terminals.jsonl");
12632        std::fs::create_dir(&path).unwrap();
12633        let supervisor = Supervisor::new_for_test(
12634            Arc::new(Registry::default()),
12635            RestartPolicy::new(0, Duration::ZERO),
12636        )
12637        .with_terminal_journal(path, "budget-journal-failure".into());
12638        let runtime = supervisor.runtime_config();
12639        let spec = windowed_crash_spec("budget-journal-failure");
12640        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12641        assert!(matches!(
12642            on_child_exit(
12643                &spec,
12644                runtime.restart_policy,
12645                &supervisor.registry,
12646                &snapshot,
12647                &runtime.terminal_ring,
12648                &runtime.spawn_events,
12649                &runtime.child_roster,
12650                crash_exit_report(1),
12651            )
12652            .await,
12653            NextAction::Stop { .. }
12654        ));
12655        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12656        let history = runtime
12657            .terminal_ring
12658            .lock()
12659            .unwrap()
12660            .durable_history(&spec.module_id);
12661        assert!(history.journal_write_failures > 0);
12662        assert_eq!(history.entries.len(), 1);
12663        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12664    }
12665
12666    #[test]
12667    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12668        let supervisor = Supervisor::new_for_test(
12669            Arc::new(Registry::default()),
12670            RestartPolicy::new(0, Duration::ZERO),
12671        );
12672        let runtime = supervisor.runtime_config();
12673        let spec = windowed_crash_spec("budget-recording-panic");
12674        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12675        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12676        // Exhausting the event sequence makes emit_exited panic before the
12677        // terminal write, exercising publication on the recording unwind.
12678        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12679        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12680            tokio::runtime::Builder::new_current_thread()
12681                .enable_all()
12682                .build()
12683                .unwrap()
12684                .block_on(on_child_exit(
12685                    &spec,
12686                    runtime.restart_policy,
12687                    &supervisor.registry,
12688                    &snapshot,
12689                    &runtime.terminal_ring,
12690                    &runtime.spawn_events,
12691                    &runtime.child_roster,
12692                    crash_exit_report(1),
12693                ))
12694        }));
12695        let panic = result.err().expect("recording must still unwind");
12696        assert_eq!(
12697            panic.downcast_ref::<String>().map(String::as_str),
12698            Some("spawn event sequence exhausted")
12699        );
12700        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12701    }
12702
12703    /// A stop the supervisor itself requests still stops a protocol-none
12704    /// module, even though the child answers the SIGTERM with exit 0.
12705    #[cfg(unix)]
12706    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12707    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12708        for disable in [false, true] {
12709            let label = if disable {
12710                "none-requested-disable"
12711            } else {
12712                "none-requested-stop"
12713            };
12714            let dir = cortexkit_test_support::ScratchDir::new(label);
12715            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12716            let supervisor = Supervisor::new_for_test(
12717                Arc::new(Registry::default()),
12718                RestartPolicy::new(3, Duration::ZERO),
12719            );
12720            let module = supervisor.spawn(spec).unwrap();
12721            wait_for_file(&ready).await;
12722
12723            if disable {
12724                module.set_enabled(false).await.unwrap();
12725            } else {
12726                module.stop().await.unwrap();
12727            }
12728            assert!(
12729                marker.exists(),
12730                "{label}: the child must have left through its SIGTERM handler with exit 0"
12731            );
12732
12733            // Long enough for a zero-backoff respawn to have happened if the
12734            // exit had been treated as a crash.
12735            sleep(Duration::from_millis(500)).await;
12736            let status = module.status().unwrap();
12737            let expected = if disable {
12738                ModuleState::Disabled
12739            } else {
12740                ModuleState::Stopped
12741            };
12742            assert_eq!(status.state, expected, "{label}");
12743            assert_eq!(
12744                status.spawn_generation, 1,
12745                "{label}: respawned after a requested stop"
12746            );
12747            let history = module.terminal_history();
12748            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12749            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12750            assert_ne!(
12751                history.entries[0].disposition,
12752                TerminalDisposition::Restarting,
12753                "{label}"
12754            );
12755        }
12756    }
12757
12758    /// A subc-wire module that exits 0 on its own is still a stop: the
12759    /// protocol-none rule must not reach it.
12760    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12761    async fn subc_wire_clean_exit_is_still_a_stop() {
12762        let supervisor = Supervisor::new_for_test(
12763            Arc::new(Registry::default()),
12764            RestartPolicy::new(3, Duration::ZERO),
12765        );
12766        let module = supervisor
12767            .spawn(ModuleSpec {
12768                module_id: "wire-clean-exit".to_string(),
12769                program: fake_aft_stub_path(),
12770                args: Vec::new(),
12771                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12772                reserved: false,
12773                reserved_prefixes: Vec::new(),
12774                protocol: ModuleProtocol::Subc,
12775                overlap: Default::default(),
12776            })
12777            .unwrap();
12778
12779        let deadline = Instant::now() + Duration::from_secs(10);
12780        while module.terminal_history().entries.is_empty() {
12781            assert!(Instant::now() < deadline, "module never exited");
12782            sleep(Duration::from_millis(10)).await;
12783        }
12784        // Long enough for a zero-backoff respawn to have happened.
12785        sleep(Duration::from_millis(500)).await;
12786        let status = module.status().unwrap();
12787        assert_eq!(status.state, ModuleState::Stopped);
12788        assert_eq!(status.spawn_generation, 1);
12789        let history = module.terminal_history();
12790        assert_eq!(history.entries.len(), 1, "{history:?}");
12791        assert_eq!(history.entries[0].exit_code, Some(0));
12792        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12793    }
12794
12795    #[cfg(unix)]
12796    #[tokio::test]
12797    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12798        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12799        let record = dir.join("live-children.json");
12800        let supervisor = Supervisor::new_for_test(
12801            Arc::new(Registry::default()),
12802            RestartPolicy::new(0, Duration::ZERO),
12803        );
12804        let mut runtime = supervisor.runtime_config();
12805        runtime.child_roster.record_to(record.clone());
12806        let gate = Arc::new(super::ReloadExitRecordGate::default());
12807        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12808        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12809        let spec = ModuleSpec {
12810            module_id: "reload-exit-roster".into(),
12811            program: fake_aft_stub_path(),
12812            args: Vec::new(),
12813            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12814            reserved: false,
12815            reserved_prefixes: Vec::new(),
12816            protocol: ModuleProtocol::Subc,
12817            overlap: Default::default(),
12818        };
12819        let mut child = None;
12820        let reload = super::finish_reload_child(
12821            &spec,
12822            &runtime,
12823            &supervisor.registry,
12824            &supervisor.process_liveness,
12825            &snapshot,
12826            &mut child,
12827        );
12828        tokio::pin!(reload);
12829        tokio::select! {
12830            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12831            _ = gate.reached.notified() => {}
12832        }
12833        assert!(runtime
12834            .terminal_ring
12835            .lock()
12836            .unwrap()
12837            .snapshot()
12838            .entries
12839            .is_empty());
12840        assert_eq!(
12841            crate::live_children::read_record(&record).unwrap().len(),
12842            1,
12843            "shutdown must still wait for the reaped child until its terminal record exists"
12844        );
12845        runtime.child_roster.close();
12846        gate.resume.notify_one();
12847        assert!(reload.await.is_err());
12848        assert!(crate::live_children::read_record(&record)
12849            .unwrap()
12850            .is_empty());
12851        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12852        assert_eq!(history.entries.len(), 1);
12853        assert_eq!(
12854            history.entries[0].disposition,
12855            TerminalDisposition::DaemonShutdown
12856        );
12857    }
12858
12859    /// Each restart-producing arm has its own state transition. Keeping their
12860    /// lifetime count assertions adjacent prevents a later new arm from silently
12861    /// spending budget without recording the historical restart.
12862    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12863    async fn every_restart_increment_path_advances_lifetime_count() {
12864        let supervisor = Supervisor::new_for_test(
12865            Arc::new(Registry::default()),
12866            RestartPolicy::new(1, Duration::ZERO),
12867        );
12868        let runtime = supervisor.runtime_config();
12869        let spec = ModuleSpec {
12870            module_id: "lifetime-increment-path".to_string(),
12871            program: PathBuf::from("/unused/lifetime-increment-path"),
12872            args: Vec::new(),
12873            env: Vec::new(),
12874            reserved: false,
12875            reserved_prefixes: Vec::new(),
12876            protocol: ModuleProtocol::Subc,
12877            overlap: Default::default(),
12878        };
12879
12880        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12881        assert!(matches!(
12882            on_child_exit(
12883                &spec,
12884                runtime.restart_policy,
12885                &supervisor.registry,
12886                &crash_snapshot,
12887                &runtime.terminal_ring,
12888                &runtime.spawn_events,
12889                &runtime.child_roster,
12890                ExitReport {
12891                    kind: ExitKind::Crash,
12892                    code: Some(1),
12893                    signal: None,
12894                    at_ms: 1,
12895                },
12896            )
12897            .await,
12898            NextAction::Restart { schedule: _ }
12899        ));
12900        let (crash_restarts, crash_lifetime) = {
12901            let state = lock_snapshot(&crash_snapshot).unwrap();
12902            (state.crash_restarts.len(), state.lifetime_restarts)
12903        };
12904        assert_eq!(crash_restarts, 1);
12905        assert_eq!(crash_lifetime, 1);
12906
12907        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12908        let mut health_child = None;
12909        assert!(matches!(
12910            health_restart_child(
12911                &spec,
12912                &runtime,
12913                &supervisor.registry,
12914                &supervisor.process_liveness,
12915                &health_snapshot,
12916                &mut health_child,
12917                SupervisorHealthStatus::Failing,
12918                None,
12919                2,
12920            )
12921            .await,
12922            Ok(())
12923        ));
12924        assert!(health_child.is_none());
12925        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12926        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12927        let (health_restarts, health_lifetime) = {
12928            let state = lock_snapshot(&health_snapshot).unwrap();
12929            (state.crash_restarts.len(), state.lifetime_restarts)
12930        };
12931        assert_eq!(health_restarts, 1);
12932        assert_eq!(health_lifetime, 1);
12933
12934        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12935        let mut reload_child = None;
12936        assert!(matches!(
12937            handle_reload_spawn_failure(
12938                &spec,
12939                &runtime,
12940                &supervisor.process_liveness,
12941                &reload_snapshot,
12942                &mut reload_child,
12943                "forced reload spawn failure".to_string(),
12944            )
12945            .await,
12946            Err(SuperviseError::ReloadFailed { .. })
12947        ));
12948        let (reload_restarts, reload_lifetime) = {
12949            let state = lock_snapshot(&reload_snapshot).unwrap();
12950            (state.crash_restarts.len(), state.lifetime_restarts)
12951        };
12952        assert_eq!(reload_restarts, 1);
12953        assert_eq!(reload_lifetime, 1);
12954    }
12955
12956    #[tokio::test]
12957    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12958        let supervisor = Supervisor::new_for_test(
12959            Arc::new(Registry::default()),
12960            RestartPolicy::new(3, Duration::ZERO),
12961        );
12962        let runtime = supervisor.runtime_config();
12963        let spec = ModuleSpec {
12964            module_id: "deliberately-severed".to_string(),
12965            program: PathBuf::from("/unused/deliberately-severed"),
12966            args: Vec::new(),
12967            env: Vec::new(),
12968            reserved: false,
12969            reserved_prefixes: Vec::new(),
12970            protocol: ModuleProtocol::Subc,
12971            overlap: Default::default(),
12972        };
12973        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12974        let process = ProcessIdentity {
12975            pid: 41,
12976            start_time: 101,
12977        };
12978        record_deliberate_severance(&snapshot, process).unwrap();
12979        let exit_report = apply_deliberate_severance_marker(
12980            &snapshot,
12981            Some(process),
12982            ExitReport {
12983                kind: ExitKind::Crash,
12984                code: Some(1),
12985                signal: None,
12986                at_ms: 1,
12987            },
12988        );
12989        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12990
12991        assert!(matches!(
12992            on_child_exit(
12993                &spec,
12994                runtime.restart_policy,
12995                &supervisor.registry,
12996                &snapshot,
12997                &runtime.terminal_ring,
12998                &runtime.spawn_events,
12999                &runtime.child_roster,
13000                exit_report,
13001            )
13002            .await,
13003            NextAction::Restart { schedule: _ }
13004        ));
13005        let state = lock_snapshot(&snapshot).unwrap();
13006        assert_eq!(state.lifetime_restarts, 1);
13007        assert_eq!(state.crash_restarts.len(), 0);
13008    }
13009
13010    #[tokio::test]
13011    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13012        let supervisor = Supervisor::new_for_test(
13013            Arc::new(Registry::default()),
13014            RestartPolicy::new(3, Duration::ZERO),
13015        );
13016        let runtime = supervisor.runtime_config();
13017        let spec = ModuleSpec {
13018            module_id: "genuine-crash".to_string(),
13019            program: PathBuf::from("/unused/genuine-crash"),
13020            args: Vec::new(),
13021            env: Vec::new(),
13022            reserved: false,
13023            reserved_prefixes: Vec::new(),
13024            protocol: ModuleProtocol::Subc,
13025            overlap: Default::default(),
13026        };
13027        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13028
13029        assert!(matches!(
13030            on_child_exit(
13031                &spec,
13032                runtime.restart_policy,
13033                &supervisor.registry,
13034                &snapshot,
13035                &runtime.terminal_ring,
13036                &runtime.spawn_events,
13037                &runtime.child_roster,
13038                ExitReport {
13039                    kind: ExitKind::Crash,
13040                    code: Some(1),
13041                    signal: None,
13042                    at_ms: 1,
13043                },
13044            )
13045            .await,
13046            NextAction::Restart { schedule: _ }
13047        ));
13048        let state = lock_snapshot(&snapshot).unwrap();
13049        assert_eq!(state.lifetime_restarts, 1);
13050        assert_eq!(state.crash_restarts.len(), 1);
13051    }
13052
13053    fn crash_exit_report(at_ms: u64) -> ExitReport {
13054        ExitReport {
13055            kind: ExitKind::Crash,
13056            code: Some(1),
13057            signal: None,
13058            at_ms,
13059        }
13060    }
13061
13062    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13063        ModuleSpec {
13064            module_id: module_id.to_string(),
13065            program: PathBuf::from("/unused").join(module_id),
13066            args: Vec::new(),
13067            env: Vec::new(),
13068            reserved: false,
13069            reserved_prefixes: Vec::new(),
13070            protocol: ModuleProtocol::Subc,
13071            overlap: Default::default(),
13072        }
13073    }
13074
13075    /// A real crash loop still stops. Three crashes with nothing aging out spend
13076    /// a budget of two and the third respawn is refused, and both surfaces an
13077    /// operator has -- the log line and the retained terminal record -- name the
13078    /// window rather than only the cap, because `max_restarts=2` alone is what
13079    /// this budget used to mean.
13080    #[tokio::test]
13081    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13082        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13083        let supervisor = Supervisor::new_for_test(
13084            Arc::new(Registry::default()),
13085            RestartPolicy::new(2, Duration::ZERO),
13086        );
13087        let runtime = supervisor.runtime_config();
13088        let spec = windowed_crash_spec("crash-loop-in-window");
13089        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13090
13091        for attempt in 1..=2 {
13092            assert!(
13093                matches!(
13094                    on_child_exit(
13095                        &spec,
13096                        runtime.restart_policy,
13097                        &supervisor.registry,
13098                        &snapshot,
13099                        &runtime.terminal_ring,
13100                        &runtime.spawn_events,
13101                        &runtime.child_roster,
13102                        crash_exit_report(attempt),
13103                    )
13104                    .await,
13105                    NextAction::Restart { schedule: _ }
13106                ),
13107                "crash {attempt} is inside the budget and must respawn"
13108            );
13109        }
13110
13111        assert!(matches!(
13112            on_child_exit(
13113                &spec,
13114                runtime.restart_policy,
13115                &supervisor.registry,
13116                &snapshot,
13117                &runtime.terminal_ring,
13118                &runtime.spawn_events,
13119                &runtime.child_roster,
13120                crash_exit_report(3),
13121            )
13122            .await,
13123            NextAction::Stop { .. }
13124        ));
13125
13126        {
13127            let state = lock_snapshot(&snapshot).unwrap();
13128            assert_eq!(state.state, ModuleState::Failed);
13129            assert_eq!(state.crash_restarts.len(), 2);
13130            assert_eq!(state.lifetime_restarts, 2);
13131        }
13132
13133        let history = runtime
13134            .terminal_ring
13135            .lock()
13136            .expect("terminal ring is not poisoned")
13137            .snapshot();
13138        let last = history
13139            .entries
13140            .last()
13141            .expect("the refused crash is retained");
13142        assert_eq!(last.disposition, TerminalDisposition::Failed);
13143        assert_eq!(
13144            last.disposition_detail.as_deref(),
13145            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13146        );
13147
13148        let captured = crate::router::test_log::captured_logs(&logs);
13149        assert!(
13150            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13151            "the stop must be logged with its window: {captured}"
13152        );
13153    }
13154
13155    /// The rate, stated as a test: three crashes where the first has aged past
13156    /// the window are two crashes as far as the budget is concerned, so the
13157    /// third respawn is allowed and the ring holds only the two recent ones.
13158    ///
13159    /// This is the case a lifetime counter got wrong -- and the case the daemon
13160    /// now hits routinely, since a module exits non-zero every time its
13161    /// connection to the daemon drops.
13162    #[tokio::test]
13163    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13164        let supervisor = Supervisor::new_for_test(
13165            Arc::new(Registry::default()),
13166            RestartPolicy::new(2, Duration::ZERO),
13167        );
13168        let runtime = supervisor.runtime_config();
13169        let spec = windowed_crash_spec("crash-across-windows");
13170        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13171
13172        for attempt in 1..=2 {
13173            assert!(matches!(
13174                on_child_exit(
13175                    &spec,
13176                    runtime.restart_policy,
13177                    &supervisor.registry,
13178                    &snapshot,
13179                    &runtime.terminal_ring,
13180                    &runtime.spawn_events,
13181                    &runtime.child_roster,
13182                    crash_exit_report(attempt),
13183                )
13184                .await,
13185                NextAction::Restart { schedule: _ }
13186            ));
13187        }
13188
13189        // The oldest crash moves out of the window; nothing else about the
13190        // module changes.
13191        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13192            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13193        })
13194        .unwrap();
13195
13196        assert!(
13197            matches!(
13198                on_child_exit(
13199                    &spec,
13200                    runtime.restart_policy,
13201                    &supervisor.registry,
13202                    &snapshot,
13203                    &runtime.terminal_ring,
13204                    &runtime.spawn_events,
13205                    &runtime.child_roster,
13206                    crash_exit_report(3),
13207                )
13208                .await,
13209                NextAction::Restart { schedule: _ }
13210            ),
13211            "a crash older than the window must not hold a budget slot"
13212        );
13213
13214        let state = lock_snapshot(&snapshot).unwrap();
13215        assert_eq!(state.state, ModuleState::Restarting);
13216        assert_eq!(
13217            state.crash_restarts.len(),
13218            2,
13219            "the aged instant is dropped and the new one takes its place"
13220        );
13221        assert_eq!(
13222            state.lifetime_restarts, 3,
13223            "the ledger counts every restart, including the ones the window forgot"
13224        );
13225    }
13226
13227    /// An operator restart hands the budget back whole, and the ledger keeps
13228    /// counting. Those are different questions -- "how close is this module to
13229    /// being stopped" and "how many times has it been replaced" -- and the
13230    /// operator action answers only the first.
13231    #[tokio::test]
13232    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13233        let supervisor = Supervisor::new_for_test(
13234            Arc::new(Registry::default()),
13235            RestartPolicy::new(2, Duration::ZERO),
13236        );
13237        let runtime = supervisor.runtime_config();
13238        let spec = windowed_crash_spec("operator-cleared-budget");
13239        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13240
13241        for attempt in 1..=2 {
13242            assert!(matches!(
13243                on_child_exit(
13244                    &spec,
13245                    runtime.restart_policy,
13246                    &supervisor.registry,
13247                    &snapshot,
13248                    &runtime.terminal_ring,
13249                    &runtime.spawn_events,
13250                    &runtime.child_roster,
13251                    crash_exit_report(attempt),
13252                )
13253                .await,
13254                NextAction::Restart { schedule: _ }
13255            ));
13256        }
13257
13258        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13259        {
13260            let state = lock_snapshot(&snapshot).unwrap();
13261            assert!(
13262                state.crash_restarts.is_empty(),
13263                "an operator restart returns the full budget"
13264            );
13265            assert_eq!(
13266                state.lifetime_restarts, 2,
13267                "clearing the budget must not unmake the crashes"
13268            );
13269        }
13270
13271        assert!(
13272            matches!(
13273                on_child_exit(
13274                    &spec,
13275                    runtime.restart_policy,
13276                    &supervisor.registry,
13277                    &snapshot,
13278                    &runtime.terminal_ring,
13279                    &runtime.spawn_events,
13280                    &runtime.child_roster,
13281                    crash_exit_report(3),
13282                )
13283                .await,
13284                NextAction::Restart { schedule: _ }
13285            ),
13286            "the cleared budget must be spendable again"
13287        );
13288        let state = lock_snapshot(&snapshot).unwrap();
13289        assert_eq!(state.crash_restarts.len(), 1);
13290        assert_eq!(state.lifetime_restarts, 3);
13291    }
13292
13293    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13294    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13295        let severed = ProcessIdentity {
13296            pid: 41,
13297            start_time: 101,
13298        };
13299        let successor = ProcessIdentity {
13300            pid: 41,
13301            start_time: 202,
13302        };
13303        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13304        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13305            state.pid = Some(successor.pid);
13306            state.process_start_time = Some(successor.start_time);
13307        })
13308        .unwrap();
13309        assert!(!module.record_deliberate_severance(severed).unwrap());
13310
13311        let exit_report = apply_deliberate_severance_marker(
13312            &module.inner.snapshot,
13313            Some(successor),
13314            ExitReport {
13315                kind: ExitKind::Crash,
13316                code: Some(1),
13317                signal: None,
13318                at_ms: 1,
13319            },
13320        );
13321
13322        assert_eq!(exit_report.kind, ExitKind::Crash);
13323    }
13324
13325    #[tokio::test]
13326    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13327        let registry = Registry::default();
13328        let supervisor = Supervisor::new_for_test(
13329            Arc::new(Registry::default()),
13330            RestartPolicy::new(3, Duration::ZERO),
13331        );
13332        let runtime = supervisor.runtime_config();
13333        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13334        let spec = ModuleSpec {
13335            module_id: "drain-deliberate-severance".to_string(),
13336            program: fake_aft_stub_path(),
13337            args: Vec::new(),
13338            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13339            reserved: false,
13340            reserved_prefixes: Vec::new(),
13341            protocol: ModuleProtocol::Subc,
13342            overlap: Default::default(),
13343        };
13344        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13345        let process = ProcessIdentity {
13346            pid: 41,
13347            start_time: 101,
13348        };
13349        child.process_identity = Some(process);
13350        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13351            state.pid = Some(process.pid);
13352            state.process_start_time = Some(process.start_time);
13353        })
13354        .unwrap();
13355        record_deliberate_severance(&snapshot, process).unwrap();
13356
13357        drain_child_to_state(
13358            &spec.module_id,
13359            spec.protocol,
13360            // The child exits on its own; no signal may change the exit this
13361            // test classifies.
13362            StopNotice::SentOverConnection,
13363            &registry,
13364            None,
13365            &snapshot,
13366            &runtime.terminal_ring,
13367            &runtime.spawn_events,
13368            child,
13369            Duration::from_secs(1),
13370            ModuleState::Stopped,
13371            Some(false),
13372        )
13373        .await
13374        .unwrap();
13375
13376        let state = lock_snapshot(&snapshot).unwrap();
13377        assert_eq!(
13378            state.last_exit.as_ref().map(|exit| exit.kind),
13379            Some(ExitKind::DeliberateSeverance)
13380        );
13381        assert_eq!(state.lifetime_restarts, 1);
13382        assert_eq!(state.crash_restarts.len(), 0);
13383        drop(state);
13384        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13385        assert_eq!(
13386            history.entries[0].exit_kind,
13387            subc_control::TerminalExitKind::DeliberateSeverance
13388        );
13389    }
13390
13391    #[tokio::test]
13392    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13393        let registry = Registry::default();
13394        let supervisor = Supervisor::new_for_test(
13395            Arc::new(Registry::default()),
13396            RestartPolicy::new(3, Duration::ZERO),
13397        );
13398        let runtime = supervisor.runtime_config();
13399        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13400        let spec = ModuleSpec {
13401            module_id: "ordinary-drain".to_string(),
13402            program: fake_aft_stub_path(),
13403            args: Vec::new(),
13404            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13405            reserved: false,
13406            reserved_prefixes: Vec::new(),
13407            protocol: ModuleProtocol::Subc,
13408            overlap: Default::default(),
13409        };
13410        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13411
13412        drain_child_to_state(
13413            &spec.module_id,
13414            spec.protocol,
13415            // The child exits on its own; no signal may change the exit this
13416            // test classifies.
13417            StopNotice::SentOverConnection,
13418            &registry,
13419            None,
13420            &snapshot,
13421            &runtime.terminal_ring,
13422            &runtime.spawn_events,
13423            child,
13424            Duration::from_secs(1),
13425            ModuleState::Stopped,
13426            Some(false),
13427        )
13428        .await
13429        .unwrap();
13430
13431        let state = lock_snapshot(&snapshot).unwrap();
13432        assert_eq!(
13433            state.last_exit.as_ref().map(|exit| exit.kind),
13434            Some(ExitKind::Crash)
13435        );
13436        assert_eq!(state.lifetime_restarts, 0);
13437        assert_eq!(state.crash_restarts.len(), 0);
13438    }
13439
13440    #[test]
13441    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13442        // The server's generic fatal-routing branch only knows that the
13443        // connection failed; it does not know that the daemon deliberately
13444        // initiated a process-killing severance. Keep this seam explicit so a
13445        // future connection error path cannot silently reintroduce the stale
13446        // exemption that mislabels a later genuine crash.
13447        assert!(!include_str!("server.rs")
13448            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13449    }
13450
13451    /// The `route.closed` `drained` value must be the quiescence wait's own
13452    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13453    /// measurement at all and `false` is the one honest constant. This is the exact
13454    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13455    /// on every return path, including the one that used to return early via `?`
13456    /// with `route.closing` already sent and no `route.closed` ever following.
13457    #[test]
13458    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13459        assert!(drained_after_quiescence_wait(&Ok(true)));
13460        assert!(!drained_after_quiescence_wait(&Ok(false)));
13461        assert!(!drained_after_quiescence_wait(&Err(
13462            SuperviseError::StatePoisoned { module_id: None }
13463        )));
13464    }
13465
13466    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13467    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13468    /// already reaped out-of-band) still leaves a terminal record rather than none
13469    /// at all. Triggering the real `wait()` I/O error from an integration test would
13470    /// need a genuine already-reaped-child race, which is OS-specific and not
13471    /// something this suite attempts elsewhere; this test instead verifies the
13472    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13473    /// the call site itself is verified by inspection to sit in that exact arm.
13474    #[test]
13475    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13476        let ring = Arc::new(Mutex::new(TerminalRing::new(
13477            TerminalRingConfig::default(),
13478            0,
13479        )));
13480        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13481
13482        let snapshot = ring.lock().unwrap().snapshot();
13483        assert_eq!(snapshot.entries.len(), 1);
13484        let entry = &snapshot.entries[0];
13485        assert_eq!(entry.exit_code, None);
13486        assert_eq!(entry.exit_signal, None);
13487        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13488    }
13489
13490    #[test]
13491    fn wait_error_exit_path_preserves_spawn_event_density() {
13492        let feed = super::SpawnEventFeed::default();
13493        feed.configure_incarnation("wait-error-density".to_string());
13494        feed.emit_spawned("wait-error", 41, 1);
13495        let ring = Arc::new(Mutex::new(TerminalRing::new(
13496            TerminalRingConfig::default(),
13497            0,
13498        )));
13499
13500        record_wait_error_terminal("wait-error", &ring, &feed);
13501        feed.emit_spawned("after-wait-error", 42, 2);
13502
13503        let state = feed.0.lock().unwrap();
13504        let sequences = state
13505            .events
13506            .iter()
13507            .map(|event| event.cursor.seq)
13508            .collect::<Vec<_>>();
13509        assert_eq!(sequences, vec![1, 2, 3]);
13510        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13511        assert_eq!(state.events[1].exit_code, None);
13512        assert_eq!(state.events[1].exit_signal, None);
13513    }
13514
13515    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13516    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13517    /// not a clean exit it never actually observed.
13518    #[test]
13519    fn wait_error_exit_report_is_classified_as_a_crash() {
13520        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13521    }
13522}
13523
13524#[cfg(all(test, unix))]
13525mod restart_ordering_tests {
13526    use super::*;
13527    use std::io::Write;
13528    use tokio::io::AsyncReadExt;
13529
13530    #[test]
13531    #[ignore = "child process fixture"]
13532    fn serving_child() {
13533        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13534            return;
13535        };
13536        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13537        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13538        for stream in listener.incoming() {
13539            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13540        }
13541    }
13542
13543    async fn restart_before_publication(requests: usize) {
13544        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13545        let address_file = home.path().join("address");
13546        let spec = ModuleSpec {
13547            module_id: "restart-serving-child".into(),
13548            program: std::env::current_exe().unwrap(),
13549            args: vec![
13550                "--exact".into(),
13551                "supervise::restart_ordering_tests::serving_child".into(),
13552                "--ignored".into(),
13553                "--nocapture".into(),
13554            ],
13555            env: vec![(
13556                "RESTART_TEST_ADDRESS_FILE".into(),
13557                address_file.display().to_string(),
13558            )],
13559            reserved: false,
13560            reserved_prefixes: vec![],
13561            protocol: ModuleProtocol::None,
13562            overlap: Default::default(),
13563        };
13564        let mut supervisor =
13565            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13566        let (spawned_tx, spawned_rx) = oneshot::channel();
13567        let spawned_tx = Mutex::new(Some(spawned_tx));
13568        let (release_tx, release_rx) = std::sync::mpsc::channel();
13569        let release_rx = Mutex::new(release_rx);
13570        // Hold publication, not the child: it must answer a real request before
13571        // the restart enters the queue, even though status still has generation 0.
13572        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13573            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13574            release_rx
13575                .lock()
13576                .unwrap()
13577                .recv_timeout(Duration::from_secs(20))
13578                .unwrap();
13579        })));
13580        let registered = supervisor
13581            .register_configured_with_health(
13582                spec.clone(),
13583                true,
13584                HealthConfig::default(),
13585                None,
13586                RestartPolicy::new(3, Duration::ZERO),
13587            )
13588            .unwrap();
13589        let module = registered.module.clone();
13590        let startup = tokio::task::spawn_blocking(move || {
13591            supervisor.start_registered_configured(registered, None)
13592        });
13593        spawned_rx.await.unwrap();
13594        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13595            let address = loop {
13596                if let Ok(address) = std::fs::read_to_string(&address_file) {
13597                    if !address.is_empty() {
13598                        break address;
13599                    }
13600                }
13601                tokio::time::sleep(Duration::from_millis(1)).await;
13602            };
13603            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13604            let mut reply = String::new();
13605            stream.read_to_string(&mut reply).await.unwrap();
13606            reply.trim().parse::<u32>().unwrap()
13607        })
13608        .await
13609        .expect("child must serve before generation publication");
13610        assert_eq!(
13611            lock_snapshot(&module.inner.snapshot)
13612                .unwrap()
13613                .spawn_generation,
13614            0
13615        );
13616        let mut replies = Vec::new();
13617        for _ in 0..requests {
13618            let mut restart = Box::pin(module.restart(Some(0)));
13619            // Poll the public API through queue admission, leaving only its ACK
13620            // pending. The monitor cannot consume the command until released.
13621            std::future::poll_fn(|cx| {
13622                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13623                std::task::Poll::Ready(())
13624            })
13625            .await;
13626            replies.push(restart);
13627        }
13628        release_tx.send(()).unwrap();
13629        startup.await.unwrap().unwrap();
13630        for reply in replies {
13631            reply.await.unwrap();
13632        }
13633        // Restart schedules its replacement after completing the handler. Wait
13634        // for that separate actor turn before placing the command barrier.
13635        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13636            loop {
13637                if lock_snapshot(&module.inner.snapshot)
13638                    .unwrap()
13639                    .spawn_generation
13640                    > 1
13641                {
13642                    break;
13643                }
13644                tokio::time::sleep(Duration::from_millis(1)).await;
13645            }
13646        })
13647        .await;
13648        // Commands are handled serially. The update-spec reply proves every
13649        // restart handler has finished before we inspect the snapshot; restart's
13650        // own ACK only acknowledges initiation, not replacement completion.
13651        module.update_spec_for_test(spec).await.unwrap();
13652        let (generation, pid) = {
13653            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13654            (state.spawn_generation, state.pid)
13655        };
13656        module.stop().await.unwrap();
13657        replacement.expect("restart must publish a replacement for the already-serving child");
13658        assert_ne!(
13659            pid,
13660            Some(old_pid),
13661            "a serving child predating the restart must be replaced"
13662        );
13663        assert_eq!(generation, 2, "queued restarts must share one replacement");
13664    }
13665
13666    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13667    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13668        restart_before_publication(1).await;
13669    }
13670
13671    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13672    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13673        restart_before_publication(2).await;
13674    }
13675
13676    #[tokio::test]
13677    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13678        let supervisor =
13679            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13680        let registered = supervisor
13681            .register_configured_with_health(
13682                ModuleSpec {
13683                    module_id: "queued-before-replacement".into(),
13684                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
13685                    // macOS a module whose running image is not the configured
13686                    // program is refused once its exec is confirmed (see
13687                    // `record_write_stall_tests::sleeping_spec`).
13688                    program: "/bin/sleep".into(),
13689                    args: vec!["600".into()],
13690                    env: vec![],
13691                    reserved: false,
13692                    reserved_prefixes: vec![],
13693                    protocol: ModuleProtocol::None,
13694                    overlap: Default::default(),
13695                },
13696                true,
13697                HealthConfig::default(),
13698                None,
13699                RestartPolicy::default(),
13700            )
13701            .unwrap();
13702        let RegisteredConfiguredModule {
13703            mut spec,
13704            mut runtime,
13705            module,
13706            mut commands,
13707        } = registered;
13708        let snapshot = &module.inner.snapshot;
13709        let mut restart = Box::pin(module.restart(Some(0)));
13710        std::future::poll_fn(|cx| {
13711            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13712            std::task::Poll::Ready(())
13713        })
13714        .await;
13715        // Drive the actor explicitly so a replacement is definitely spawned
13716        // after receipt, but before this queued command is dequeued.
13717        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13718        let replacement_pid = child.as_ref().unwrap().id();
13719        let command = commands.recv().await.unwrap();
13720        assert!(
13721            handle_supervisor_command(
13722                command,
13723                &mut spec,
13724                &mut runtime,
13725                &supervisor.registry,
13726                &supervisor.process_liveness,
13727                snapshot,
13728                &mut child,
13729                &mut commands,
13730                &mut VecDeque::new(),
13731            )
13732            .await
13733        );
13734        restart.await.unwrap();
13735        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13736        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13737        if let Some(mut child) = child {
13738            child.start_kill().unwrap();
13739            child.wait().await.unwrap();
13740        }
13741        assert_eq!(
13742            kept_pid, replacement_pid,
13743            "a newer replacement must survive"
13744        );
13745        assert!(
13746            !pending,
13747            "a satisfied restart must not schedule another spawn"
13748        );
13749    }
13750}
13751
13752#[cfg(all(test, unix))]
13753mod health_event_tests {
13754    use super::*;
13755    use cortexkit_test_support::ScratchDir;
13756    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13757
13758    struct Actor {
13759        module: SupervisedModule,
13760        registry: Arc<Registry>,
13761        forwarding: Arc<ForwardingTable>,
13762        spec: ModuleSpec,
13763        health: HealthConfig,
13764        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13765        _home: ScratchDir,
13766    }
13767
13768    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13769        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13770        // for an observed condition, rather than letting paused time auto-advance
13771        // deadlines or assuming a fixed number of yields finishes the work.
13772        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13773        let tick = Instant::now();
13774        loop {
13775            if let Some(value) = observe() {
13776                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13777                return value;
13778            }
13779            assert!(
13780                std::time::Instant::now() < deadline,
13781                "timed out waiting for {reason}"
13782            );
13783            tokio::task::yield_now().await;
13784        }
13785    }
13786
13787    impl Actor {
13788        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13789            let home = ScratchDir::new("health-events");
13790            let registry = Arc::new(Registry::default());
13791            let forwarding = Arc::new(ForwardingTable::default());
13792            let health = HealthConfig {
13793                cadence,
13794                deadline: Duration::from_secs(3600),
13795                ..HealthConfig::default()
13796            };
13797            let supervisor =
13798                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13799                    .with_forwarding(forwarding.clone())
13800                    .with_health_config(health.clone());
13801            let spec = ModuleSpec {
13802                module_id: "event-health-child".into(),
13803                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
13804                // the privacy trampoline refuses a module whose running image
13805                // is not the configured program, and `start` below waits for
13806                // the process to be confirmed (`reported_pid`). The shell form
13807                // passed only while the image check ran before the shell had
13808                // replaced itself with `sleep` (see
13809                // `record_write_stall_tests::sleeping_spec`).
13810                program: PathBuf::from("/bin/sleep"),
13811                // The process stays alive but has no real bus peer. Registrations
13812                // below exercise the same registry writes as accepted HELLOs.
13813                args: vec!["600".into()],
13814                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13815                    .into_iter()
13816                    .map(|key| (key.into(), home.path().display().to_string()))
13817                    .collect(),
13818                reserved: false,
13819                reserved_prefixes: vec![],
13820                protocol,
13821                overlap: Default::default(),
13822            };
13823            let module = supervisor.spawn(spec.clone()).unwrap();
13824            let (_, rx) = mpsc::channel(8);
13825            let actor = Self {
13826                module,
13827                registry,
13828                forwarding,
13829                spec,
13830                health,
13831                rx,
13832                _home: home,
13833            };
13834            actor
13835                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13836                .await;
13837            actor
13838        }
13839
13840        fn turns(&self) -> u64 {
13841            lock_snapshot(&self.module.inner.snapshot)
13842                .unwrap()
13843                .actor_turns
13844        }
13845
13846        async fn wait_for_select(
13847            &self,
13848            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13849        ) -> ActorSelectCheckpoint {
13850            wait_for("actor select checkpoint", || {
13851                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13852                state.actor_select.filter(|checkpoint| {
13853                    checkpoint.generation == state.spawn_generation
13854                        && state.state == ModuleState::Running
13855                        && state.process_alive
13856                        && state.reported_pid().is_some()
13857                        && expected(checkpoint)
13858                })
13859            })
13860            .await
13861        }
13862
13863        fn hello(&mut self, connection: u64, health: bool) {
13864            let manifest =
13865                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13866                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13867                    .build();
13868            let connection = ConnectionId::new(connection);
13869            let (tx, rx) = mpsc::channel(8);
13870            self.rx = rx;
13871            self.forwarding
13872                .register_module_connection(
13873                    connection,
13874                    self.spec.module_id.clone(),
13875                    subc_protocol::PROTOCOL_VERSION,
13876                    Concurrency::ModuleManaged,
13877                    FrameSink::new(tx),
13878                )
13879                .unwrap();
13880            self.registry
13881                .register_with_control_ops(
13882                    manifest,
13883                    subc_protocol::PROTOCOL_VERSION,
13884                    connection,
13885                    if health {
13886                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13887                    } else {
13888                        vec![]
13889                    },
13890                )
13891                .unwrap();
13892        }
13893
13894        async fn probe(&mut self) -> crate::router::OutboundFrame {
13895            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13896                Ok(frame) => Some(frame),
13897                Err(mpsc::error::TryRecvError::Empty) => None,
13898                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13899            })
13900            .await;
13901            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13902            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13903            frame
13904        }
13905
13906        fn no_probe(&mut self) {
13907            assert!(matches!(
13908                self.rx.try_recv(),
13909                Err(mpsc::error::TryRecvError::Empty)
13910            ));
13911        }
13912
13913        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13914            self.forwarding
13915                .complete_module_control_rpc(
13916                    ConnectionId::new(connection),
13917                    frame.header.corr,
13918                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13919                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13920                        status: HealthStatus::Ok,
13921                        detail: None,
13922                        metrics: None,
13923                    }),
13924                )
13925                .unwrap();
13926        }
13927    }
13928
13929    #[tokio::test(start_paused = true)]
13930    async fn no_health_children_park_for_sixty_seconds() {
13931        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13932        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13933        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13934        unadvertised.hello(1, false);
13935        // Include any one-off registration wake in the budget; receiving a
13936        // no-health HELLO must not turn a parked child into a polling child.
13937        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13938        // a single 60 s jump would only observe one expired timer.
13939        for _ in 0..6000 {
13940            tokio::time::advance(Duration::from_millis(10)).await;
13941            tokio::task::yield_now().await;
13942        }
13943        for actor in [&none, &unregistered, &unadvertised] {
13944            assert!(
13945                actor.turns() <= 5,
13946                "no-health actor woke {} times",
13947                actor.turns()
13948            );
13949            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13950        }
13951    }
13952
13953    #[tokio::test(start_paused = true)]
13954    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13955        // With a zero cadence the first probe is due at once, so this measures
13956        // only how fast the registration notification starts health probing,
13957        // not the normal 30-33 s wait before a first probe (checked below).
13958        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13959        let tick = Instant::now();
13960        actor.hello(1, true);
13961        actor.probe().await;
13962        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13963    }
13964
13965    #[tokio::test(start_paused = true)]
13966    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13967        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13968        let hello_tick = Instant::now();
13969        actor.hello(1, true);
13970        let armed = actor
13971            .wait_for_select(|checkpoint| {
13972                checkpoint.registered_connection == Some(ConnectionId::new(1))
13973                    && checkpoint.next_probe_at.is_some()
13974            })
13975            .await;
13976        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13977        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13978        let first_deadline = hello_tick + delay;
13979        assert_eq!(armed.next_probe_at, Some(first_deadline));
13980        tokio::time::advance(Duration::from_secs(29)).await;
13981        let turns = actor.turns();
13982        actor
13983            .registry
13984            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13985            .unwrap();
13986        let updated = actor
13987            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13988            .await;
13989        assert_eq!(
13990            updated.next_probe_at,
13991            Some(first_deadline),
13992            "catalog update must not reset cadence"
13993        );
13994        actor.no_probe();
13995        tokio::time::advance(first_deadline - Instant::now()).await;
13996        let frame = actor.probe().await;
13997        actor.answer(1, frame);
13998        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13999        let next_deadline = Instant::now() + next;
14000        let rearmed = actor
14001            .wait_for_select(|checkpoint| {
14002                checkpoint
14003                    .next_probe_at
14004                    .is_some_and(|deadline| deadline > first_deadline)
14005            })
14006            .await;
14007        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14008        tokio::time::advance(next - Duration::from_nanos(1)).await;
14009        actor.no_probe();
14010        tokio::time::advance(Duration::from_nanos(1)).await;
14011        actor.probe().await;
14012    }
14013
14014    #[tokio::test(start_paused = true)]
14015    async fn disconnect_disarms_and_reconnect_rearms_health() {
14016        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14017        actor.hello(1, true);
14018        actor
14019            .wait_for_select(|checkpoint| {
14020                checkpoint.registered_connection == Some(ConnectionId::new(1))
14021                    && checkpoint.next_probe_at.is_some()
14022            })
14023            .await;
14024        let turns = actor.turns();
14025        let tick = Instant::now();
14026        actor
14027            .registry
14028            .deregister_connection(ConnectionId::new(1))
14029            .unwrap();
14030        let disarmed = actor
14031            .wait_for_select(|checkpoint| {
14032                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14033            })
14034            .await;
14035        assert_eq!(disarmed.next_probe_at, None);
14036        assert_eq!(disarmed.wake_after, None);
14037        assert_eq!(
14038            Instant::now(),
14039            tick,
14040            "disconnect must disarm in the notification tick"
14041        );
14042        tokio::time::advance(Duration::from_secs(60)).await;
14043        actor.no_probe();
14044        let reconnect_tick = Instant::now();
14045        actor.hello(2, true);
14046        let rearmed = actor
14047            .wait_for_select(|checkpoint| {
14048                checkpoint.registered_connection == Some(ConnectionId::new(2))
14049                    && checkpoint.next_probe_at.is_some()
14050            })
14051            .await;
14052        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14053        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14054        tokio::time::advance(delay).await;
14055        actor.probe().await;
14056    }
14057
14058    #[tokio::test(start_paused = true)]
14059    async fn reregistration_without_health_disarms_the_old_schedule() {
14060        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14061        actor.hello(1, true);
14062        actor
14063            .wait_for_select(|checkpoint| {
14064                checkpoint.registered_connection == Some(ConnectionId::new(1))
14065                    && checkpoint.next_probe_at.is_some()
14066            })
14067            .await;
14068        actor
14069            .registry
14070            .deregister_connection(ConnectionId::new(1))
14071            .unwrap();
14072        actor.hello(2, false);
14073        let disarmed = actor
14074            .wait_for_select(|checkpoint| {
14075                checkpoint.registered_connection == Some(ConnectionId::new(2))
14076            })
14077            .await;
14078        assert_eq!(disarmed.next_probe_at, None);
14079        assert_eq!(disarmed.wake_after, None);
14080        tokio::time::advance(Duration::from_secs(60)).await;
14081        actor.no_probe();
14082    }
14083
14084    #[tokio::test(start_paused = true)]
14085    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14086        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14087        actor.hello(1, true);
14088        let armed = actor
14089            .wait_for_select(|checkpoint| {
14090                checkpoint.registered_connection == Some(ConnectionId::new(1))
14091                    && checkpoint.next_probe_at.is_some()
14092            })
14093            .await;
14094        // Simulate the old process's connection closing before the restart
14095        // tears the process down. The replacement process has started but has
14096        // not sent its HELLO, so it is not registered.
14097        actor
14098            .registry
14099            .deregister_connection(ConnectionId::new(1))
14100            .unwrap();
14101        actor.module.restart(Some(0)).await.unwrap();
14102        let parked = actor
14103            .wait_for_select(|checkpoint| {
14104                checkpoint.generation > armed.generation
14105                    && checkpoint.registered_connection.is_none()
14106                    && checkpoint.next_probe_at.is_none()
14107            })
14108            .await;
14109        assert_eq!(parked.wake_after, None);
14110        actor.rx.close();
14111        // Drain the stop notice the restart queued for the old process's connection,
14112        // so it isn't mistaken for traffic to the replacement.
14113        while actor.rx.try_recv().is_ok() {}
14114        let turns = actor.turns();
14115        tokio::time::advance(Duration::from_secs(60)).await;
14116        let after = actor.turns();
14117        eprintln!(
14118            "restart park turns: before={turns} after={after} delta={}",
14119            after - turns
14120        );
14121        assert_eq!(after, turns, "replacement must park before HELLO");
14122        let hello_tick = Instant::now();
14123        actor.hello(2, true);
14124        let rearmed = actor
14125            .wait_for_select(|checkpoint| {
14126                checkpoint.registered_connection == Some(ConnectionId::new(2))
14127                    && checkpoint.next_probe_at.is_some()
14128            })
14129            .await;
14130        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14131        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14132        tokio::time::advance(delay).await;
14133        actor.probe().await;
14134    }
14135
14136    #[tokio::test(start_paused = true)]
14137    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14138        use tokio::{
14139            io::{AsyncReadExt, AsyncWriteExt},
14140            net::TcpListener,
14141        };
14142        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14143        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14144        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14145        let (started_tx, mut started_rx) = mpsc::channel(8);
14146        let peer = tokio::spawn(async move {
14147            loop {
14148                let (mut socket, _) = listener.accept().await.unwrap();
14149                let mut request = [0; 1024];
14150                assert!(socket.read(&mut request).await.unwrap() > 0);
14151                started_tx.send(Instant::now()).await.unwrap();
14152                socket
14153                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14154                    .await
14155                    .unwrap();
14156            }
14157        });
14158        actor
14159            .module
14160            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14161            .await
14162            .unwrap();
14163        let first_tick = Instant::now();
14164        let first = actor
14165            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14166            .await;
14167        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14168        assert!(started_rx.try_recv().is_err());
14169
14170        // Restart just before the old deadline: reusing it would give the new
14171        // process only one second to start, rather than a full cadence.
14172        tokio::time::advance(
14173            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14174        )
14175        .await;
14176        actor.module.restart(Some(0)).await.unwrap();
14177        let spawn_tick = Instant::now();
14178        let replacement = actor
14179            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14180            .await;
14181        let deadline = replacement.next_probe_at.unwrap();
14182        assert!(
14183            deadline >= spawn_tick + actor.health.cadence,
14184            "a replacement must not inherit its predecessor's probe deadline"
14185        );
14186        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14187        assert!(
14188            started_rx.try_recv().is_err(),
14189            "no HTTP probe during startup grace"
14190        );
14191        tokio::time::advance(deadline - Instant::now()).await;
14192        let landed = wait_for("first replacement HTTP probe", || {
14193            started_rx.try_recv().ok()
14194        })
14195        .await;
14196        assert!(landed >= spawn_tick + actor.health.cadence);
14197        wait_for("healthy replacement", || {
14198            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14199                .then_some(())
14200        })
14201        .await;
14202        peer.abort();
14203    }
14204
14205    #[tokio::test(start_paused = true)]
14206    async fn rescan_adds_and_removes_http_health_without_polling() {
14207        use tokio::{
14208            io::{AsyncReadExt, AsyncWriteExt},
14209            net::TcpListener,
14210        };
14211        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14212        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14213        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14214        // One millisecond separates successive probes without a polling delay.
14215        actor.health.cadence = Duration::from_millis(1);
14216        let (started_tx, mut started_rx) = mpsc::channel(8);
14217        let peer = tokio::spawn(async move {
14218            loop {
14219                let (mut socket, _) = listener.accept().await.unwrap();
14220                let mut request = [0; 1024];
14221                assert!(socket.read(&mut request).await.unwrap() > 0);
14222                started_tx.send(()).await.unwrap();
14223                socket
14224                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14225                    .await
14226                    .unwrap();
14227            }
14228        });
14229        let tick = Instant::now();
14230        let turns = actor.turns();
14231        actor
14232            .module
14233            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14234            .await
14235            .unwrap();
14236        let armed = actor
14237            .wait_for_select(|checkpoint| {
14238                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14239            })
14240            .await;
14241        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14242        assert_eq!(Instant::now(), tick);
14243        tokio::time::advance(Duration::from_millis(1)).await;
14244        let mut started = false;
14245        wait_for("successful HTTP probe", || {
14246            started |= started_rx.try_recv().is_ok();
14247            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14248                .then_some(())
14249        })
14250        .await;
14251        assert!(started, "added HTTP check must start at its first deadline");
14252        assert_eq!(
14253            actor.module.status().unwrap().health.status,
14254            SupervisorHealthStatus::Ok
14255        );
14256        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14257        actor.health.http = None;
14258        let turns = actor.turns();
14259        actor
14260            .module
14261            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14262            .await
14263            .unwrap();
14264        let parked = actor
14265            .wait_for_select(|checkpoint| {
14266                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14267            })
14268            .await;
14269        assert_eq!(parked.wake_after, None);
14270        let turns = actor.turns();
14271        tokio::time::advance(Duration::from_secs(60)).await;
14272        assert_eq!(
14273            actor.turns(),
14274            turns,
14275            "removed HTTP check must park the actor"
14276        );
14277        assert!(
14278            started_rx.try_recv().is_err(),
14279            "removed HTTP check must stay disarmed"
14280        );
14281        assert_eq!(
14282            actor.module.status().unwrap().health,
14283            ModuleHealthStatus::default()
14284        );
14285        peer.abort();
14286    }
14287}
14288
14289#[cfg(test)]
14290mod drain_busy_reprobe_tests {
14291    //! The drain's busy-gauge wait, driven against an in-process module
14292    //! connection whose health answers the test scripts. Time is paused, so
14293    //! every elapsed duration below is exact virtual time.
14294    use super::*;
14295    use std::collections::BTreeMap;
14296    use std::sync::Mutex as StdMutex;
14297    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14298
14299    const MODULE_ID: &str = "drain-reprobe-module";
14300    const CONNECTION: u64 = 7;
14301    const GAUGE: &str = "runs_in_flight";
14302    /// The production default for both the health cadence and the drain
14303    /// ceiling, which is the pairing that used to cost a full ceiling.
14304    const CADENCE: Duration = Duration::from_secs(30);
14305    const CEILING: Duration = Duration::from_secs(30);
14306
14307    /// How the scripted module answers each `health.check`.
14308    #[derive(Clone, Copy)]
14309    enum Answer {
14310        /// Report the gauge busy until this instant, quiet from then on, after
14311        /// taking `latency` to answer.
14312        BusyUntil { until: Instant, latency: Duration },
14313        /// Answer at once, with metrics that leave the declared gauge out.
14314        OmitsGauge,
14315        /// Never answer.
14316        Silent,
14317    }
14318
14319    struct Drain {
14320        forwarding: Arc<ForwardingTable>,
14321        runtime: SupervisorRuntimeConfig,
14322        endpoint: crate::ModuleEndpointId,
14323        /// When each probe reached the module.
14324        probes: Arc<StdMutex<Vec<Instant>>>,
14325        responder: tokio::task::JoinHandle<()>,
14326    }
14327
14328    impl Drain {
14329        fn start(answer: Answer) -> Self {
14330            let registry = Arc::new(Registry::default());
14331            let forwarding = Arc::new(ForwardingTable::default());
14332            let health = HealthConfig {
14333                cadence: CADENCE,
14334                ..HealthConfig::default()
14335            };
14336            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14337                .with_forwarding(forwarding.clone())
14338                .with_health_config(health)
14339                .runtime_config();
14340            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14341            let endpoint = forwarding
14342                .register_module_connection(
14343                    ConnectionId::new(CONNECTION),
14344                    MODULE_ID.to_string(),
14345                    subc_protocol::PROTOCOL_VERSION,
14346                    Concurrency::ModuleManaged,
14347                    FrameSink::new(tx),
14348                )
14349                .unwrap();
14350            let probes = Arc::new(StdMutex::new(Vec::new()));
14351            let responder = tokio::spawn({
14352                let forwarding = forwarding.clone();
14353                let probes = probes.clone();
14354                async move {
14355                    while let Some(frame) = rx.recv().await {
14356                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14357                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14358                        probes.lock().unwrap().push(Instant::now());
14359                        let metrics = match answer {
14360                            Answer::BusyUntil { until, latency } => {
14361                                sleep(latency).await;
14362                                let busy = u64::from(Instant::now() < until);
14363                                serde_json::json!({ GAUGE: busy })
14364                            }
14365                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14366                            Answer::Silent => continue,
14367                        };
14368                        // A probe abandoned at its deadline has no RPC left to
14369                        // complete, so a late answer may be refused.
14370                        let _ = forwarding.complete_module_control_rpc(
14371                            ConnectionId::new(CONNECTION),
14372                            frame.header.corr,
14373                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14374                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14375                                status: HealthStatus::Ok,
14376                                detail: None,
14377                                metrics: Some(metrics),
14378                            }),
14379                        );
14380                    }
14381                }
14382            });
14383            Self {
14384                forwarding,
14385                runtime,
14386                endpoint,
14387                probes,
14388                responder,
14389            }
14390        }
14391
14392        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14393        /// how long it took.
14394        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14395            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14396            (drained, elapsed)
14397        }
14398
14399        /// [`Self::run`], also returning what the drain's probes read.
14400        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14401            let started = Instant::now();
14402            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14403            let drained = wait_for_forwarding_quiescence(
14404                &self.forwarding,
14405                MODULE_ID,
14406                &self.runtime,
14407                self.endpoint,
14408                started + ceiling,
14409                &[GAUGE.to_string()],
14410                DrainScope::Active,
14411                &mut trace,
14412            )
14413            .await
14414            .unwrap();
14415            (drained, started.elapsed(), trace)
14416        }
14417
14418        fn probes(&self) -> Vec<Instant> {
14419            self.probes.lock().unwrap().clone()
14420        }
14421    }
14422
14423    impl Drop for Drain {
14424        fn drop(&mut self) {
14425            self.responder.abort();
14426        }
14427    }
14428
14429    #[tokio::test(start_paused = true)]
14430    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14431        let drain = Drain::start(Answer::BusyUntil {
14432            until: Instant::now() + Duration::from_secs(1),
14433            latency: Duration::ZERO,
14434        });
14435        let (drained, elapsed) = drain.run(CEILING).await;
14436        assert!(
14437            drained,
14438            "gauges that read zero during the drain must let it finish drained, \
14439             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14440        );
14441        assert!(
14442            elapsed < Duration::from_secs(3),
14443            "drain took {elapsed:?} after the gauges went quiet at 1s"
14444        );
14445        let probes = drain.probes();
14446        assert!(
14447            probes.len() >= 2,
14448            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14449        );
14450    }
14451
14452    /// The same shape in real time on a multi-threaded runtime: the first
14453    /// probe is answered busy at once, within a millisecond of the drain
14454    /// starting, and later probes read idle. The drain must keep probing
14455    /// after that first instant answer and end soon after the gauge falls.
14456    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14457    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14458        let drain = Drain::start(Answer::BusyUntil {
14459            until: Instant::now() + Duration::from_millis(600),
14460            latency: Duration::ZERO,
14461        });
14462        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14463        assert!(drained, "the gauge fell at 600ms: {trace}");
14464        assert!(
14465            elapsed < Duration::from_secs(3),
14466            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14467        );
14468        let probes = drain.probes();
14469        assert!(
14470            probes.len() >= 2 && trace.probes as usize == probes.len(),
14471            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14472        );
14473    }
14474
14475    #[tokio::test(start_paused = true)]
14476    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14477        let started = Instant::now();
14478        let drain = Drain::start(Answer::BusyUntil {
14479            until: started + Duration::from_secs(3600),
14480            latency: Duration::ZERO,
14481        });
14482        let (drained, elapsed) = drain.run(CEILING).await;
14483        assert!(!drained, "a module busy throughout must not drain");
14484        assert!(
14485            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14486            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14487        );
14488        // Re-probing is paced by the interval, not a busy loop: at most one
14489        // probe per interval across the ceiling, plus the first at time zero.
14490        let probes = drain.probes();
14491        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14492        assert!(
14493            probes.len() as u128 <= max_probes,
14494            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14495            probes.len()
14496        );
14497        for pair in probes.windows(2) {
14498            assert!(
14499                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14500                "consecutive probes {:?} apart",
14501                pair[1] - pair[0]
14502            );
14503        }
14504    }
14505
14506    #[tokio::test(start_paused = true)]
14507    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14508        // Each answer takes longer than the re-probe interval; the next probe
14509        // must wait for the previous answer and then the interval.
14510        let latency = Duration::from_secs(1);
14511        let drain = Drain::start(Answer::BusyUntil {
14512            until: Instant::now() + Duration::from_secs(3600),
14513            latency,
14514        });
14515        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14516        assert!(!drained);
14517        let probes = drain.probes();
14518        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14519        for pair in probes.windows(2) {
14520            assert!(
14521                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14522                "a probe started {:?} after the previous one, before it could finish",
14523                pair[1] - pair[0]
14524            );
14525        }
14526    }
14527
14528    #[tokio::test(start_paused = true)]
14529    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14530        let drain = Drain::start(Answer::Silent);
14531        let ceiling = Duration::from_secs(2);
14532        let (drained, elapsed) = drain.run(ceiling).await;
14533        assert!(!drained, "an unanswered probe counts as busy");
14534        assert!(
14535            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14536            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14537        );
14538        assert_eq!(
14539            drain.probes().len(),
14540            1,
14541            "the unanswered probe holds the only slot until the ceiling"
14542        );
14543    }
14544
14545    /// Every event logged while it is installed, as `message` plus fields.
14546    #[derive(Clone, Default)]
14547    struct LogCapture {
14548        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14549    }
14550
14551    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14552        fn on_event(
14553            &self,
14554            event: &tracing::Event<'_>,
14555            _context: tracing_subscriber::layer::Context<'_, S>,
14556        ) {
14557            struct Fields(BTreeMap<String, String>);
14558            impl tracing::field::Visit for Fields {
14559                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14560                    self.0
14561                        .insert(field.name().to_string(), format!("{value:?}"));
14562                }
14563            }
14564            let mut fields = Fields(BTreeMap::new());
14565            event.record(&mut fields);
14566            self.events.lock().unwrap().push(fields.0);
14567        }
14568    }
14569
14570    impl LogCapture {
14571        fn install(&self) -> tracing::subscriber::DefaultGuard {
14572            use tracing_subscriber::layer::SubscriberExt;
14573            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14574        }
14575
14576        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14577            self.events
14578                .lock()
14579                .unwrap()
14580                .iter()
14581                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14582                .cloned()
14583                .collect()
14584        }
14585    }
14586
14587    /// A module that declares a busy gauge but leaves it out of its health
14588    /// answers holds a drain to its ceiling. The timeout line must say so by
14589    /// name, and the drain must warn about the missing gauge once, not on
14590    /// every probe.
14591    #[tokio::test(start_paused = true)]
14592    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14593        let capture = LogCapture::default();
14594        let _guard = capture.install();
14595        let drain = Drain::start(Answer::OmitsGauge);
14596        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14597        assert!(!drained, "a gauge never reported cannot read quiet");
14598        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14599
14600        let missing = capture.with_message("lacks a declared busy gauge");
14601        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14602        assert_eq!(
14603            missing[0].get("missing_gauges").map(String::as_str),
14604            Some(GAUGE)
14605        );
14606
14607        warn_drain_timed_out(
14608            MODULE_ID,
14609            Duration::from_secs(2),
14610            RouteCloseReason::Restart,
14611            &Default::default(),
14612            &trace,
14613        );
14614        let timeouts = capture.with_message("route drain timed out");
14615        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14616        let gauges = &timeouts[0]["busy_gauges"];
14617        assert!(
14618            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14619            "the timeout line must name the gauge and why it read busy: {gauges}"
14620        );
14621        assert!(
14622            gauges.ends_with("; no value ever read"),
14623            "no value was ever read for the gauge: {gauges}"
14624        );
14625        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14626    }
14627
14628    /// The trace names a busy value and when it was read. For a probe that
14629    /// never got an answer, it names the probe's error kind and says that no
14630    /// value was ever read.
14631    #[tokio::test(start_paused = true)]
14632    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14633        let busy = Drain::start(Answer::BusyUntil {
14634            until: Instant::now() + Duration::from_secs(3600),
14635            latency: Duration::ZERO,
14636        });
14637        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14638        let rendered = trace.to_string();
14639        assert!(
14640            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14641                && rendered.contains("; last value non_zero(1) @"),
14642            "{rendered}"
14643        );
14644
14645        let silent = Drain::start(Answer::Silent);
14646        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14647        // The probe is capped at the 1 s ceiling, and its reading is stamped
14648        // when it gave up.
14649        assert_eq!(
14650            trace.to_string(),
14651            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14652        );
14653        assert_eq!(trace.probes, 1);
14654    }
14655
14656    /// A gauge value that is not a count reads busy and is shown as sent.
14657    #[test]
14658    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14659        let gauges = [GAUGE.to_string()];
14660        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14661        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14662        assert_eq!(
14663            busy_gauge_observation(&verdicts),
14664            BusyGaugeObservation::Busy
14665        );
14666        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14667        assert_eq!(
14668            busy_gauge_observation(&zero),
14669            BusyGaugeObservation::Quiescent
14670        );
14671        let none = gauge_verdicts(None, &gauges);
14672        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14673    }
14674}
14675
14676#[cfg(test)]
14677mod health_evidence_tests {
14678    use super::{HealthProbeError, HealthProbeEvidence};
14679    use std::collections::HashSet;
14680
14681    /// The evidential asymmetry, asserted rather than described.
14682    ///
14683    /// Exactly ONE observation is proof a module cannot serve, and the one that
14684    /// fires under CPU starvation is not it. Before the split, all fifteen
14685    /// construction sites collapsed into a single String, so a timeout carried the
14686    /// same weight as a dead lane -- which is how a healthy module was restarted
14687    /// three times in one day.
14688    #[test]
14689    fn only_a_dead_lane_is_proof_of_death() {
14690        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14691        // Three non-proof classes, each for a different reason: silence is
14692        // consistent with health, a bad answer proves the module ALIVE, and a
14693        // daemon-side fault never reached the module at all.
14694        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14695        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14696        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14697    }
14698
14699    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14700    ///
14701    /// A shared label renders two different observations identically in the line an
14702    /// operator reads after an unexplained restart -- the exact confusion this
14703    /// change removes.
14704    #[test]
14705    fn every_evidence_class_has_a_distinct_label() {
14706        let labels = [
14707            HealthProbeError::lane_dead("").label(),
14708            HealthProbeError::no_answer("").label(),
14709            HealthProbeError::bad_answer("").label(),
14710            HealthProbeError::misconfigured("").label(),
14711        ];
14712        let unique: HashSet<_> = labels.iter().collect();
14713        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14714    }
14715
14716    /// The class is additional information, not a replacement.
14717    ///
14718    /// An operator needs both "this was silence" and the specific text saying how
14719    /// long we waited; a classification that swallowed the message would trade one
14720    /// missing distinction for another.
14721    #[test]
14722    fn classification_preserves_the_original_message() {
14723        let err = HealthProbeError::no_answer("module did not answer within 5s");
14724        assert_eq!(err.to_string(), "module did not answer within 5s");
14725        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14726    }
14727}
14728
14729#[cfg(test)]
14730mod health_tombstone_tests {
14731    use std::{path::PathBuf, sync::Arc, time::Duration};
14732
14733    use subc_protocol::{
14734        manifest::Concurrency,
14735        session::{HealthStatus, ModuleControlResponse},
14736    };
14737    use tokio::sync::mpsc;
14738
14739    use super::{
14740        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14741        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14742    };
14743    use crate::{
14744        control::ControlHandler,
14745        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14746        registry::{ConnectionId, Registry},
14747        router::FrameSink,
14748    };
14749
14750    struct ProbeHarness {
14751        spec: ModuleSpec,
14752        runtime: SupervisorRuntimeConfig,
14753        forwarding: Arc<ForwardingTable>,
14754        module_connection: ConnectionId,
14755        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14756        handler: ControlHandler,
14757        module: super::SupervisedModule,
14758    }
14759
14760    fn probe_harness() -> ProbeHarness {
14761        let registry = Arc::new(Registry::default());
14762        let forwarding = Arc::new(ForwardingTable::default());
14763        let supervisor_handle = super::SupervisorHandle::new();
14764        let health = HealthConfig {
14765            http: None,
14766            cadence: Duration::from_secs(30),
14767            deadline: Duration::from_secs(5),
14768            failure_threshold: 3,
14769            on_degraded: HealthAction::Report,
14770            on_failing: HealthAction::Report,
14771            critical: false,
14772        };
14773        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14774            .with_forwarding(Arc::clone(&forwarding))
14775            .with_handle(supervisor_handle.clone())
14776            .with_health_config(health);
14777        let spec = ModuleSpec {
14778            module_id: "late-health-module".to_string(),
14779            program: PathBuf::from("disabled-module"),
14780            args: Vec::new(),
14781            env: Vec::new(),
14782            reserved: false,
14783            reserved_prefixes: Vec::new(),
14784            protocol: ModuleProtocol::Subc,
14785            overlap: Default::default(),
14786        };
14787        let module = supervisor
14788            .supervise_configured(spec.clone(), false)
14789            .unwrap();
14790        let runtime = supervisor.runtime_config();
14791        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14792            .with_supervisor(supervisor_handle);
14793        let module_connection = ConnectionId::new(700);
14794        let (module_tx, module_rx) = mpsc::channel(8);
14795        forwarding
14796            .register_module_connection(
14797                module_connection,
14798                spec.module_id.clone(),
14799                subc_protocol::PROTOCOL_VERSION,
14800                Concurrency::ModuleManaged,
14801                FrameSink::new(module_tx),
14802            )
14803            .unwrap();
14804
14805        ProbeHarness {
14806            spec,
14807            runtime,
14808            forwarding,
14809            module_connection,
14810            module_rx,
14811            handler,
14812            module,
14813        }
14814    }
14815
14816    async fn finish_after(
14817        harness: &mut ProbeHarness,
14818        stall: Duration,
14819    ) -> ModuleControlRpcCompletion {
14820        assert!(stall > harness.runtime.health.deadline);
14821        let deadline = harness.runtime.health.deadline;
14822        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14823        let answer = async {
14824            let frame = harness.module_rx.recv().await.expect("health.check frame");
14825            tokio::time::advance(deadline).await;
14826            tokio::task::yield_now().await;
14827            tokio::time::advance(stall - deadline).await;
14828            harness
14829                .forwarding
14830                .complete_module_control_rpc(
14831                    harness.module_connection,
14832                    frame.header.corr,
14833                    Some("health.check"),
14834                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14835                        status: HealthStatus::Ok,
14836                        detail: None,
14837                        metrics: None,
14838                    }),
14839                )
14840                .unwrap()
14841        };
14842        let (probe_result, completion) = tokio::join!(probe, answer);
14843        let err = probe_result.expect_err("probe must miss its deadline");
14844        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14845        completion
14846    }
14847
14848    async fn time_out_without_answer(harness: &mut ProbeHarness) {
14849        let deadline = harness.runtime.health.deadline;
14850        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14851        let exhaust_deadline = async {
14852            let _frame = harness.module_rx.recv().await.expect("health.check frame");
14853            tokio::time::advance(deadline).await;
14854            tokio::task::yield_now().await;
14855        };
14856        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14857        let err = probe_result.expect_err("probe must miss its deadline");
14858        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14859    }
14860
14861    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14862        let registry = Arc::clone(&harness.module.inner.registry);
14863        let snapshot = Arc::clone(&harness.module.inner.snapshot);
14864        let process_liveness = super::SupervisorProcessLiveness::default();
14865        let mut child = None;
14866        let cycle = super::run_health_probe_cycle(
14867            &harness.spec,
14868            &harness.runtime,
14869            &registry,
14870            &process_liveness,
14871            &snapshot,
14872            &mut child,
14873        );
14874        let peer = async {
14875            let frame = harness.module_rx.recv().await.expect("health.check frame");
14876            if answer {
14877                harness
14878                    .forwarding
14879                    .complete_module_control_rpc(
14880                        harness.module_connection,
14881                        frame.header.corr,
14882                        Some("health.check"),
14883                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14884                            status: HealthStatus::Ok,
14885                            detail: None,
14886                            metrics: Some(serde_json::json!({"ready": true})),
14887                        }),
14888                    )
14889                    .unwrap();
14890            } else {
14891                tokio::time::advance(harness.runtime.health.deadline).await;
14892                tokio::task::yield_now().await;
14893            }
14894        };
14895        tokio::join!(cycle, peer);
14896    }
14897
14898    #[tokio::test(start_paused = true)]
14899    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14900        let mut harness = probe_harness();
14901        // Drive the probe cycle directly with an in-memory wire peer. Stop the
14902        // disabled module's monitor so only this test owns lifecycle transitions;
14903        // no OS process is launched, and a restart is observed at scheduling.
14904        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14905        monitor.abort();
14906        let _ = monitor.await;
14907        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14908            state.enabled = true;
14909            state.state = super::ModuleState::Running;
14910            state.process_alive = true;
14911        })
14912        .unwrap();
14913
14914        run_probe_cycle(&mut harness, true).await;
14915        assert_eq!(
14916            harness.module.status().unwrap().health.status,
14917            super::SupervisorHealthStatus::Ok
14918        );
14919
14920        for failures in 1..harness.runtime.health.failure_threshold {
14921            run_probe_cycle(&mut harness, false).await;
14922            let status = harness.module.status().unwrap();
14923            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14924            assert_eq!(status.health.consecutive_failures, failures);
14925            assert!(status.health.last_probe_ms.is_some());
14926            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14927            assert!(status.health.metrics.is_none());
14928            assert_eq!(status.state, super::ModuleState::Running);
14929            assert!(status.process_alive);
14930            assert_eq!(status.restart_count, 0);
14931            assert_eq!(status.lifetime_restarts, 0);
14932            assert!(status.health.last_action.is_none());
14933            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14934        }
14935
14936        run_probe_cycle(&mut harness, true).await;
14937        let recovered = harness.module.status().unwrap();
14938        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14939        assert_eq!(recovered.health.consecutive_failures, 0);
14940        assert!(recovered.health.detail.is_none());
14941        assert_eq!(
14942            recovered.health.metrics,
14943            Some(serde_json::json!({"ready": true}))
14944        );
14945        assert_eq!(recovered.lifetime_restarts, 0);
14946
14947        for failures in 1..=harness.runtime.health.failure_threshold {
14948            run_probe_cycle(&mut harness, false).await;
14949            let status = harness.module.status().unwrap();
14950            assert_eq!(status.health.consecutive_failures, failures);
14951            if failures < harness.runtime.health.failure_threshold {
14952                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14953                assert_eq!(status.state, super::ModuleState::Running);
14954                assert_eq!(status.lifetime_restarts, 0);
14955                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14956            } else {
14957                assert_eq!(
14958                    status.health.status,
14959                    super::SupervisorHealthStatus::Unresponsive
14960                );
14961                assert_eq!(status.state, super::ModuleState::Restarting);
14962                assert_eq!(status.restart_count, 1);
14963                assert_eq!(status.lifetime_restarts, 1);
14964                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14965                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14966            }
14967        }
14968    }
14969
14970    #[tokio::test(start_paused = true)]
14971    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14972        let mut harness = probe_harness();
14973
14974        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14975        let first_latency = match &first {
14976            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14977            other => panic!("late answer was not retained: {other:?}"),
14978        };
14979        assert!(harness.handler.observe_module_control_completion(first));
14980
14981        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14982        let second_latency = match &second {
14983            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14984            other => panic!("late answer was not retained: {other:?}"),
14985        };
14986        assert!(harness.handler.observe_module_control_completion(second));
14987
14988        assert_eq!(first_latency, Duration::from_secs(8));
14989        assert_eq!(
14990            second_latency - first_latency,
14991            Duration::from_secs(3),
14992            "latency must grow linearly with the additional stall"
14993        );
14994        let health = harness.module.status().unwrap().health;
14995        assert_eq!(health.late_answer_count, 2);
14996        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14997    }
14998
14999    /// A module that answers every probe late must never march to the kill
15000    /// threshold: the late answer proves it is alive, so it must clear the miss
15001    /// streak the timeout recorded. Without the reset, a CPU-starved module
15002    /// that serves every probe seconds past the deadline accumulates
15003    /// `consecutive_failures` to the threshold and is killed — the exact
15004    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15005    /// "proves the module is alive" five times while counting five misses.
15006    #[tokio::test(start_paused = true)]
15007    async fn late_answer_clears_the_consecutive_failure_streak() {
15008        let mut harness = probe_harness();
15009
15010        // Timeout recorded first: the probe path saw no answer in time.
15011        time_out_without_answer(&mut harness).await;
15012        harness
15013            .module
15014            .record_health_probe_failure_for_test("[no-answer] test miss")
15015            .unwrap();
15016        assert_eq!(
15017            harness.module.status().unwrap().health.consecutive_failures,
15018            1,
15019            "precondition: the miss must be on the streak before the late answer"
15020        );
15021
15022        // The stalled reply then lands: proof of life.
15023        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15024        assert!(matches!(
15025            late,
15026            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15027        ));
15028        assert!(harness.handler.observe_module_control_completion(late));
15029
15030        let health = harness.module.status().unwrap().health;
15031        assert_eq!(
15032            health.consecutive_failures, 0,
15033            "a late answer is an answer: the streak must reset"
15034        );
15035        assert_eq!(health.late_answer_count, 1);
15036    }
15037
15038    #[tokio::test(start_paused = true)]
15039    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15040        let mut harness = probe_harness();
15041
15042        for _ in 0..20 {
15043            time_out_without_answer(&mut harness).await;
15044            assert_eq!(
15045                harness.forwarding.health_probe_tombstone_count().unwrap(),
15046                1
15047            );
15048        }
15049    }
15050}
15051
15052#[cfg(test)]
15053mod child_env_tests {
15054    #[cfg(windows)]
15055    use super::{
15056        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15057    };
15058    #[cfg(windows)]
15059    use std::sync::{Arc, OnceLock};
15060    #[cfg(windows)]
15061    #[test]
15062    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15063        let spec = spec(Vec::new());
15064        let mut command = Command::new("unused");
15065        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15066            .unwrap()
15067            .unwrap();
15068        let name = command
15069            .as_std()
15070            .get_envs()
15071            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15072            .and_then(|(_, value)| value)
15073            .unwrap();
15074        assert_eq!(name, OsStr::new(handoff.name()));
15075        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15076        assert!(command.as_std().get_envs().any(|(key, value)| key
15077            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15078            && value == Some(OsStr::new("env"))));
15079        assert!(command
15080            .as_std()
15081            .get_envs()
15082            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15083    }
15084
15085    #[cfg(windows)]
15086    #[test]
15087    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15088        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15089        snapshot.process_alive = true;
15090        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15091        assert_eq!(
15092            observed_launch_nonce_source(&snapshot).as_deref(),
15093            Some("env")
15094        );
15095        let consumed = Arc::new(OnceLock::new());
15096        snapshot.pipe_consumed = Some(consumed.clone());
15097        assert_eq!(
15098            observed_launch_nonce_source(&snapshot).as_deref(),
15099            Some("env")
15100        );
15101        consumed.set(()).unwrap();
15102        assert_eq!(
15103            observed_launch_nonce_source(&snapshot).as_deref(),
15104            Some("pipe")
15105        );
15106        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15107        assert_eq!(
15108            observed_launch_nonce_source(&snapshot).as_deref(),
15109            Some("env")
15110        );
15111        clear_current_process_facts(&mut snapshot);
15112        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15113    }
15114    use super::{
15115        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15116        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15117        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15118    };
15119    use std::{ffi::OsStr, path::PathBuf};
15120    use tokio::process::Command;
15121
15122    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15123        ModuleSpec {
15124            module_id: "env-plan".to_string(),
15125            program: PathBuf::from("/nonexistent"),
15126            args: Vec::new(),
15127            env,
15128            reserved: false,
15129            reserved_prefixes: Vec::new(),
15130            protocol: ModuleProtocol::Subc,
15131            overlap: Default::default(),
15132        }
15133    }
15134
15135    #[test]
15136    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15137        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15138        let digest = LaunchSpecDigest::from_spec(&base);
15139        let mut reordered = base.clone();
15140        reordered.env.reverse();
15141        reordered.module_id = "different-id".into();
15142        reordered.reserved = true;
15143        reordered.overlap = Default::default();
15144        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15145        let mut delimiter_collision = base.clone();
15146        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15147        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15148        let mut changed = base.clone();
15149        changed.protocol = ModuleProtocol::None;
15150        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15151        changed = base.clone();
15152        changed.args.push("--flag".into());
15153        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15154        changed = base.clone();
15155        changed.program = PathBuf::from("/other");
15156        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15157        changed = base.clone();
15158        changed.env[1].1 = "other".into();
15159        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15160        assert!(!format!("{digest:?}").contains("B=C"));
15161        assert!(!format!("{digest:?}").contains("[u8"));
15162    }
15163
15164    #[cfg(unix)]
15165    #[test]
15166    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15167        use std::os::unix::ffi::OsStringExt;
15168        let mut left = spec(Vec::new());
15169        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15170        let mut right = left.clone();
15171        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15172        assert_eq!(
15173            left.program.to_string_lossy(),
15174            right.program.to_string_lossy()
15175        );
15176        assert_ne!(
15177            LaunchSpecDigest::from_spec(&left),
15178            LaunchSpecDigest::from_spec(&right)
15179        );
15180    }
15181
15182    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
15183    /// one still gets its own.
15184    ///
15185    /// This is the narrow goal `env_clear()` was reached for, and the reason the
15186    /// fix is `env_remove` rather than deleting the line: an operator's ambient
15187    /// filter silently becoming an unconfigured module's log level is a real
15188    /// defect, just a much smaller one than clearing the environment.
15189    ///
15190    /// Asserted on the command plan rather than a spawned child because proving
15191    /// the ABSENCE of an inherited variable needs the parent's environment
15192    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
15193    /// removal as `(key, None)`, which is exactly the distinction wanted: not
15194    /// "absent because nobody set it" but "explicitly unset for the child".
15195    #[test]
15196    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15197        let mut command = Command::new("/nonexistent");
15198        apply_child_env(&mut command, &spec(Vec::new()));
15199        let removed = command
15200            .as_std()
15201            .get_envs()
15202            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15203        assert!(
15204            removed,
15205            "ambient CK_LOG must be explicitly removed for an unconfigured module"
15206        );
15207
15208        let mut configured = Command::new("/nonexistent");
15209        apply_child_env(
15210            &mut configured,
15211            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15212        );
15213        let effective = configured
15214            .as_std()
15215            .get_envs()
15216            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15217            .last()
15218            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15219        assert_eq!(
15220            effective,
15221            Some(Some("debug".to_string())),
15222            "a module's configured CK_LOG must survive the ambient removal"
15223        );
15224    }
15225
15226    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
15227    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
15228    /// the same reason as the CK_LOG test above.
15229    ///
15230    /// The argument is the load-bearing half: a stock binary exits on an
15231    /// unknown flag before it listens, so with `--subc` appended the mode
15232    /// could not supervise the one process it exists for. Found by the first
15233    /// conformance run (nats-server: `flag provided but not defined: -subc`).
15234    #[test]
15235    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15236        let connection_file = std::path::Path::new("/run/subc-connection.json");
15237        let handle = SupervisorHandle::new();
15238
15239        let mut none_spec = spec(Vec::new());
15240        none_spec.protocol = ModuleProtocol::None;
15241        let mut none = Command::new("/nonexistent");
15242        let none_handoff =
15243            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15244                .expect("protocol-none spawn args apply");
15245        assert!(
15246            none_handoff.is_none(),
15247            "protocol:none spawn must not receive a nonce descriptor"
15248        );
15249        #[cfg(windows)]
15250        assert!(!none.as_std().get_envs().any(|(key, value)| key
15251            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15252            && value.is_some()));
15253        assert!(
15254            !none.as_std().get_envs().any(|(key, value)| key
15255                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15256                && value.is_some()),
15257            "protocol:none spawn must not name a nonce descriptor"
15258        );
15259        let none_args: Vec<String> = none
15260            .as_std()
15261            .get_args()
15262            .map(|a| a.to_string_lossy().into_owned())
15263            .collect();
15264        assert!(
15265            !none_args.iter().any(|a| a == SUBC_ARG),
15266            "protocol:none argv must not carry --subc; got {none_args:?}"
15267        );
15268        let none_has_nonce = none
15269            .as_std()
15270            .get_envs()
15271            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15272        assert!(
15273            !none_has_nonce,
15274            "protocol:none spawn must not receive a launch nonce"
15275        );
15276        let none_has_module_id = none
15277            .as_std()
15278            .get_envs()
15279            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15280        assert!(
15281            none_has_module_id,
15282            "SUBC_MODULE_ID is inert and stays on every path"
15283        );
15284        assert!(
15285            handle.spawn_nonce(&none_spec.module_id).is_none(),
15286            "no nonce record for a process that will never present one"
15287        );
15288
15289        // Control: the subc-wire path is unchanged by the branch above.
15290        let wire_spec = spec(Vec::new());
15291        let mut wire = Command::new("/nonexistent");
15292        let wire_handoff =
15293            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15294                .expect("subc-wire spawn args apply");
15295        let wire_fd_env = wire
15296            .as_std()
15297            .get_envs()
15298            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15299            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15300        #[cfg(unix)]
15301        assert_eq!(
15302            wire_fd_env,
15303            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15304            "a subc-wire spawn names the pipe it will receive at descriptor 3"
15305        );
15306        #[cfg(windows)]
15307        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15308        #[cfg(not(any(unix, windows)))]
15309        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15310        let wire_args: Vec<String> = wire
15311            .as_std()
15312            .get_args()
15313            .map(|a| a.to_string_lossy().into_owned())
15314            .collect();
15315        assert_eq!(
15316            wire_args,
15317            vec![
15318                SUBC_ARG.to_string(),
15319                connection_file.to_string_lossy().into_owned()
15320            ],
15321            "a subc-wire spawn still carries --subc <path>"
15322        );
15323        assert_eq!(
15324            wire.as_std()
15325                .get_envs()
15326                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15327            !cfg!(unix),
15328            "only Windows supplies the environment nonce"
15329        );
15330        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15331    }
15332
15333    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
15334    /// spec tries to set it; only a swap candidate carries it.
15335    ///
15336    /// "Set it only on candidates" is not enough, because spawn applies the
15337    /// spec's env verbatim and the daemon's own environment is inherited: either
15338    /// could hand a plain restart the swap role, and a module reading it would
15339    /// warm on its long swap budget while callers wait. Asserted as an explicit
15340    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
15341    /// test above gives.
15342    #[test]
15343    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15344        let role = |command: &Command| {
15345            command
15346                .as_std()
15347                .get_envs()
15348                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15349                .last()
15350                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15351        };
15352        let forged = spec(vec![(
15353            SUBC_SPAWN_ROLE_ENV.to_string(),
15354            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15355        )]);
15356
15357        let mut plain = Command::new("/nonexistent");
15358        apply_child_env(&mut plain, &forged);
15359        apply_spawn_role(&mut plain, SpawnRole::Plain);
15360        assert_eq!(
15361            role(&plain),
15362            Some(None),
15363            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15364        );
15365
15366        let mut candidate = Command::new("/nonexistent");
15367        apply_child_env(&mut candidate, &spec(Vec::new()));
15368        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15369        assert_eq!(
15370            role(&candidate),
15371            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15372        );
15373    }
15374
15375    /// Daemon-private capture retention keys never reach the child.
15376    ///
15377    /// cortexkit-log exposes retention as a Rust struct with no environment
15378    /// names, so these entries are supervisor metadata. Passing them through
15379    /// would invent a public child-process contract by accident.
15380    #[test]
15381    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15382        let mut command = Command::new("/nonexistent");
15383        apply_child_env(
15384            &mut command,
15385            &spec(vec![
15386                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15387                ("KEPT".to_string(), "yes".to_string()),
15388            ]),
15389        );
15390        let keys: Vec<String> = command
15391            .as_std()
15392            .get_envs()
15393            .filter(|(_, value)| value.is_some())
15394            .map(|(key, _)| key.to_string_lossy().into_owned())
15395            .collect();
15396        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15397        assert!(
15398            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15399            "daemon-private capture key leaked to the child: {keys:?}"
15400        );
15401    }
15402}
15403
15404#[cfg(test)]
15405mod jitter_tests {
15406    use super::jittered_health_delay;
15407    use std::{collections::HashSet, time::Duration};
15408
15409    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15410    /// that actually occur rather than invented ones.
15411    ///
15412    /// This is a SAMPLE, not a registry: the property under test is that distinct
15413    /// ids disperse, which holds for any set of distinct strings. Several entries
15414    /// are already historical (modules get renamed), and that costs nothing here --
15415    /// but it means a reader must not mistake this for the live module set, and a
15416    /// rename sweep will match it without there being anything to change.
15417    const FLEET: [&str; 14] = [
15418        "aft",
15419        "alfonso-core",
15420        "magic-context",
15421        "broca",
15422        "thalamus",
15423        "quota",
15424        "engram",
15425        "plexus",
15426        "cerebellum",
15427        "astrocyte",
15428        "synapse",
15429        "subc-mcp",
15430        "cortexkit-credentials",
15431        "subc-federation",
15432    ];
15433
15434    /// Probes must not converge after a fleet-wide restart.
15435    ///
15436    /// This is the property the jitter exists for: every module reconnects at
15437    /// once, and without dispersal all fourteen would then probe on the same
15438    /// tick forever. Nothing failed visibly when this went untested -- a
15439    /// convergent fleet still probes correctly, just in a burst, so the symptom
15440    /// is a periodic load spike that looks like whatever else is running.
15441    #[test]
15442    fn probe_delays_disperse_across_the_fleet() {
15443        let cadence = Duration::from_secs(30);
15444        let delays: HashSet<Duration> = FLEET
15445            .iter()
15446            .map(|id| jittered_health_delay(id, 0, cadence))
15447            .collect();
15448        assert_eq!(
15449            delays.len(),
15450            FLEET.len(),
15451            "every supervised module must land on its own probe offset"
15452        );
15453    }
15454
15455    /// The offset may only ever DELAY a probe, never bring it forward.
15456    ///
15457    /// A delay below the cadence would probe a module more often than
15458    /// configured, which is the opposite of what an operator asked for and
15459    /// would tighten the failure budget without anyone changing it.
15460    #[test]
15461    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15462        let cadence = Duration::from_secs(30);
15463        let span = cadence / 10;
15464        for id in FLEET {
15465            for probe_index in 0..8 {
15466                let delay = jittered_health_delay(id, probe_index, cadence);
15467                assert!(
15468                    delay >= cadence,
15469                    "{id}#{probe_index}: jitter must not shorten the cadence"
15470                );
15471                assert!(
15472                    delay < cadence + span,
15473                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15474                );
15475            }
15476        }
15477    }
15478
15479    /// A module keeps its offset across daemon restarts.
15480    ///
15481    /// The delay is derived rather than randomised precisely so a restart does
15482    /// not re-roll every module into a fresh chance of collision. A random
15483    /// source would satisfy the dispersal test above and quietly lose this.
15484    #[test]
15485    fn a_module_offset_is_stable_across_restarts() {
15486        let cadence = Duration::from_secs(30);
15487        for id in FLEET {
15488            assert_eq!(
15489                jittered_health_delay(id, 0, cadence),
15490                jittered_health_delay(id, 0, cadence),
15491                "{id}: the same module and probe index must produce the same offset"
15492            );
15493        }
15494    }
15495
15496    /// A zero cadence disables probing rather than producing a busy loop.
15497    #[test]
15498    fn zero_cadence_yields_zero_delay() {
15499        assert_eq!(
15500            jittered_health_delay("aft", 0, Duration::ZERO),
15501            Duration::ZERO
15502        );
15503    }
15504}
15505
15506#[cfg(all(test, target_os = "linux"))]
15507mod cgroup_placement_tests {
15508    use super::{
15509        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15510        SuperviseError, SupervisedChild,
15511    };
15512    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15513    use cortexkit_test_support::ScratchDir;
15514    use std::{
15515        fs, io,
15516        path::{Path, PathBuf},
15517        sync::{Arc, Mutex},
15518    };
15519    use tokio::process::Command;
15520
15521    #[tokio::test]
15522    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15523        use super::*;
15524        let dir = ScratchDir::new("unique-spawn-cgroups");
15525        let root = PathBuf::from(format!(
15526            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15527            std::process::id(),
15528            unix_ms_now()
15529        ));
15530        if let Err(error) = fs::create_dir(&root) {
15531            assert!(
15532                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15533                "required cgroup test cannot execute: {error}"
15534            );
15535            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15536            return;
15537        }
15538        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15539        let group_count = || {
15540            fs::read_dir(root.join("subc-modules"))
15541                .unwrap()
15542                .map(|entry| entry.unwrap().file_type().unwrap())
15543                .filter(|kind| kind.is_dir())
15544                .count()
15545        };
15546        let supervisor =
15547            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15548                .with_cgroup_placement(Some(placement.clone()));
15549        let runtime = supervisor.runtime_config();
15550        let mut spec = ModuleSpec {
15551            module_id: "unique-spawn".into(),
15552            program: PathBuf::from("/bin/sleep"),
15553            args: vec!["60".into()],
15554            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15555                .into_iter()
15556                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15557                .collect(),
15558            reserved: false,
15559            reserved_prefixes: vec![],
15560            protocol: ModuleProtocol::None,
15561            overlap: Default::default(),
15562        };
15563        let spawn = |spec: &ModuleSpec| {
15564            spawn_child(
15565                spec,
15566                None,
15567                None,
15568                &runtime.stderr_ring,
15569                None,
15570                &runtime.child_roster,
15571                Some(&placement),
15572            )
15573            .unwrap()
15574        };
15575        let mut live = spawn(&spec);
15576        for _ in 0..3 {
15577            // A new process can enter the old slot while retirement is pending.
15578            let next = spawn(&spec);
15579            assert_ne!(live.module_id, next.module_id);
15580            live.start_kill().unwrap();
15581            live.wait().await.unwrap();
15582            live = next;
15583            assert!(
15584                live.child.try_wait().unwrap().is_none(),
15585                "retiring the old slot must not kill the replacement"
15586            );
15587            assert_eq!(
15588                group_count(),
15589                1,
15590                "only the live spawn's cgroup should remain"
15591            );
15592        }
15593        supervisor.begin_daemon_shutdown();
15594        let reap = tokio::spawn(async move {
15595            live.wait().await.unwrap();
15596        });
15597        supervisor
15598            .end_children_for_daemon_shutdown(false, std::future::pending())
15599            .await;
15600        reap.await.unwrap();
15601        assert_eq!(group_count(), 0);
15602        // A normal exit uses the same tree-cleanup path as a killed spawn.
15603        spec.program = PathBuf::from("/bin/true");
15604        spec.args.clear();
15605        let fresh_roster = ChildRoster::default();
15606        let mut short = spawn_child(
15607            &spec,
15608            None,
15609            None,
15610            &runtime.stderr_ring,
15611            None,
15612            &fresh_roster,
15613            Some(&placement),
15614        )
15615        .unwrap();
15616        short.wait().await.unwrap();
15617        assert_eq!(group_count(), 0);
15618        spec.module_id = "_".repeat(255);
15619        let mut long_id = spawn_child(
15620            &spec,
15621            None,
15622            None,
15623            &runtime.stderr_ring,
15624            None,
15625            &fresh_roster,
15626            Some(&placement),
15627        )
15628        .unwrap();
15629        long_id.wait().await.unwrap();
15630        assert_eq!(
15631            group_count(),
15632            0,
15633            "valid long module IDs must not exceed cgroup NAME_MAX"
15634        );
15635        fs::remove_dir(root.join("subc-modules")).unwrap();
15636        fs::remove_dir(root).unwrap();
15637        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15638    }
15639
15640    #[test]
15641    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15642        let path = Path::new("/definitely-missing-subc-cgroup");
15643        let mut command = Command::new("true");
15644        let error = apply_cgroup_placement(
15645            &mut command,
15646            &ModuleSpec {
15647                module_id: "broken-cgroup".to_string(),
15648                program: PathBuf::from("true"),
15649                args: Vec::new(),
15650                env: Vec::new(),
15651                reserved: false,
15652                reserved_prefixes: Vec::new(),
15653                protocol: ModuleProtocol::Subc,
15654                overlap: Default::default(),
15655            },
15656            path,
15657        )
15658        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15659        let reason = error.to_string();
15660
15661        assert!(
15662            matches!(error, SuperviseError::Cgroup { .. }),
15663            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15664        );
15665        assert!(
15666            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15667            "parent cgroup open failure must name cgroup.procs: {reason}"
15668        );
15669    }
15670
15671    #[tokio::test]
15672    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15673        let root = ScratchDir::new("supervisor-reap-cgroup");
15674        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15675        let placement = subc_cgroup::prepare_at(&root)
15676            .expect("prepare scratch cgroup root")
15677            .expect("scratch root has a cgroup.procs marker");
15678        let module_id = "reaped-module";
15679        let module = placement
15680            .module_path(module_id)
15681            .expect("create scratch module cgroup");
15682        let child = Command::new("true")
15683            .env("XDG_DATA_HOME", root.path())
15684            .env("XDG_RUNTIME_DIR", root.path())
15685            .env("XDG_CONFIG_HOME", root.path())
15686            .spawn()
15687            .expect("spawn short-lived child");
15688        let pid = child.id().expect("spawned child has pid");
15689        let mut child = SupervisedChild {
15690            child,
15691            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15692            protocol: ModuleProtocol::Subc,
15693            module_id: module_id.to_string(),
15694            cgroup_placement: Some(placement),
15695            stdout_pump: None,
15696            stderr_pump: None,
15697            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15698            spawned_at: tokio::time::Instant::now(),
15699            spawned_at_ms: 0,
15700            spawned_from: PathBuf::from("true"),
15701            launch_spec_digest: LaunchSpecDigest([0; 32]),
15702            spawned_file_identity: None,
15703            process_start_time: None,
15704            process_identity: None,
15705            pid,
15706            roster_guard: None,
15707            #[cfg(target_os = "macos")]
15708            privacy_exec: None,
15709            spawn_failure: None,
15710        };
15711
15712        child.wait().await.expect("reap short-lived child");
15713
15714        assert!(
15715            !module.exists(),
15716            "reaping the supervised child must remove its empty cgroup"
15717        );
15718    }
15719
15720    #[test]
15721    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15722        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15723        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15724        let placement = subc_cgroup::prepare_at(&root)
15725            .expect("prepare scratch cgroup root")
15726            .expect("scratch root has a cgroup.procs marker");
15727        let module = placement
15728            .module_path("surviving-module")
15729            .expect("create scratch module cgroup");
15730        fs::write(module.join("surviving-process"), b"still present")
15731            .expect("make scratch cgroup non-empty");
15732        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15733
15734        remove_module_cgroup(&placement, "surviving-module");
15735
15736        let logs = crate::router::test_log::captured_logs(&logs);
15737        assert!(
15738            module.exists(),
15739            "failed removal must leave the cgroup intact"
15740        );
15741        assert!(
15742            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15743                && logs.contains("surviving-module"),
15744            "best-effort removal must report the failure without returning it: {logs}"
15745        );
15746    }
15747
15748    #[test]
15749    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15750        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15751        let reason = SuperviseError::Spawn {
15752            program: PathBuf::from("/bin/true"),
15753            source: io::Error::from_raw_os_error(13),
15754            cgroup_path: Some(cgroup_path.clone()),
15755        }
15756        .to_string();
15757
15758        assert!(
15759            reason.contains(&cgroup_path.display().to_string()),
15760            "a pre_exec spawn failure must name the cgroup path: {reason}"
15761        );
15762    }
15763}
15764
15765#[cfg(test)]
15766mod spawn_subscriber_lag_tests {
15767    use super::*;
15768
15769    /// A subscriber whose connection stops draining is dropped once its frame
15770    /// channel fills. The client must learn that from a terminal Error frame
15771    /// after the frames already queued for it, not from a stream that simply
15772    /// goes quiet.
15773    #[tokio::test]
15774    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15775        let feed = SpawnEventFeed::default();
15776        feed.configure_incarnation("lag-incarnation".to_string());
15777        // A one-slot connection queue that nobody reads until the emits are
15778        // done: the forwarder parks on it and the subscriber channel fills.
15779        let (tx, mut rx) = mpsc::channel(1);
15780        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15781            .expect("subscribe");
15782        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15783        for index in 0..emitted {
15784            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15785            // Let the forwarder take what it can so the fill point is the
15786            // subscriber channel, not a scheduling accident.
15787            tokio::task::yield_now().await;
15788        }
15789        assert_eq!(
15790            feed.subscriber_count(),
15791            0,
15792            "the lagged subscriber must be removed"
15793        );
15794
15795        let mut data = Vec::new();
15796        let mut last = None;
15797        loop {
15798            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15799                .await
15800                .expect("the forwarder must finish once the subscriber is dropped");
15801            let Some(outbound) = next else { break };
15802            let frame = outbound.frame;
15803            if frame.header.ty == FrameType::StreamData {
15804                assert!(last.is_none(), "no data may follow the terminal frame");
15805                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15806                data.push(event.cursor.seq);
15807            } else {
15808                assert!(last.is_none(), "exactly one terminal frame");
15809                last = Some(frame);
15810            }
15811        }
15812        assert!(!data.is_empty(), "queued frames drain before the terminal");
15813        for pair in data.windows(2) {
15814            assert_eq!(
15815                pair[1],
15816                pair[0] + 1,
15817                "queued frames arrive dense and in order"
15818            );
15819        }
15820        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15821        assert_eq!(terminal.header.ty, FrameType::Error);
15822        assert_eq!(terminal.header.corr, 7);
15823        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15824        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15825        let detail = body.detail.expect("lagged error carries detail");
15826        assert_eq!(
15827            detail["first_undelivered_cursor"]["seq"],
15828            data.last().unwrap() + 1,
15829            "the named cursor is the first event the subscriber did not receive"
15830        );
15831        assert_eq!(
15832            detail["first_undelivered_cursor"]["daemon_incarnation"],
15833            "lag-incarnation"
15834        );
15835    }
15836}
15837
15838#[cfg(test)]
15839mod terminal_history_read_concurrency_tests {
15840    use super::*;
15841    use crate::terminal_journal::read_pause;
15842    use cortexkit_test_support::ScratchDir;
15843    use std::sync::mpsc as std_mpsc;
15844
15845    fn journaled_ring(
15846        journal: &Arc<crate::terminal_journal::TerminalJournal>,
15847    ) -> Arc<Mutex<TerminalRing>> {
15848        Arc::new(Mutex::new(
15849            TerminalRing::new(TerminalRingConfig::default(), 1)
15850                .with_journal(Some(Arc::clone(journal))),
15851        ))
15852    }
15853
15854    fn crash(at_ms: u64) -> ExitReport {
15855        ExitReport {
15856            kind: ExitKind::Crash,
15857            code: Some(1),
15858            signal: None,
15859            at_ms,
15860        }
15861    }
15862
15863    /// Record an exit on another thread and report whether it finished within
15864    /// `bound`. The recorder thread is left running if it did not.
15865    fn record_within(
15866        module_id: &'static str,
15867        ring: &Arc<Mutex<TerminalRing>>,
15868        at_ms: u64,
15869        bound: Duration,
15870    ) -> bool {
15871        let ring = Arc::clone(ring);
15872        let (done, done_rx) = std_mpsc::channel();
15873        std::thread::spawn(move || {
15874            record_terminal(
15875                module_id,
15876                &ring,
15877                &SpawnEventFeed::default(),
15878                &crash(at_ms),
15879                TerminalDisposition::Restarting,
15880            );
15881            let _ = done.send(());
15882        });
15883        done_rx.recv_timeout(bound).is_ok()
15884    }
15885
15886    /// A history read in progress must not hold the journal writer (which every
15887    /// module's exit recording needs) or the module's own ring. Exits recorded
15888    /// while the read is paused complete promptly; the paused read answers as of
15889    /// the moment it started, and the next read has each exit exactly once.
15890    #[test]
15891    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15892        let dir = ScratchDir::new("terminal-history-concurrent-read");
15893        let path = dir.join("terminals.jsonl");
15894        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15895            path.clone(),
15896            "daemon".into(),
15897        ));
15898        let reader_ring = journaled_ring(&journal);
15899        let other_ring = journaled_ring(&journal);
15900        assert!(record_within(
15901            "reader-module",
15902            &reader_ring,
15903            10,
15904            Duration::from_secs(5)
15905        ));
15906
15907        let (started, release) = read_pause::install(&path);
15908        let reading = {
15909            let ring = Arc::clone(&reader_ring);
15910            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15911        };
15912        started
15913            .recv_timeout(Duration::from_secs(5))
15914            .expect("the history read reached its pause");
15915
15916        let bound = Duration::from_secs(1);
15917        assert!(
15918            record_within("other-module", &other_ring, 20, bound),
15919            "another module's exit waited on a history read (journal writer held)"
15920        );
15921        assert!(
15922            record_within("reader-module", &reader_ring, 30, bound),
15923            "the read module's own exit waited on its history read (ring held)"
15924        );
15925
15926        drop(release);
15927        let paused = reading.join().unwrap();
15928        assert_eq!(
15929            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15930            vec![10],
15931            "an exit recorded after the read began lands in neither half of it"
15932        );
15933        assert_eq!(paused.journal_skipped_lines, 0);
15934        assert_eq!(paused.journal_read_errors, 0);
15935
15936        let after = durable_terminal_history_of(&reader_ring, "reader-module");
15937        assert_eq!(
15938            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15939            vec![10, 30],
15940            "the next read merges ring and journal with no duplicate"
15941        );
15942        assert_eq!(after.journal_skipped_lines, 0);
15943    }
15944}
15945
15946/// What a restart does with the exited process's output readers. These drive
15947/// the same settle functions the supervisor calls, with a reader the test
15948/// holds, so a reader that has not been scheduled by the bound is a controlled
15949/// input rather than something only a loaded machine produces.
15950#[cfg(test)]
15951mod stderr_settle_tests {
15952    use std::{
15953        future::Future,
15954        io,
15955        pin::Pin,
15956        sync::{Arc, Mutex},
15957        task::{Context, Poll},
15958        time::Duration,
15959    };
15960
15961    use tokio::{
15962        io::{AsyncRead, ReadBuf},
15963        sync::oneshot,
15964        time::Instant,
15965    };
15966
15967    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15968    use crate::stderr_tail::{
15969        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15970        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15971    };
15972
15973    const BOUND: Duration = Duration::from_millis(250);
15974
15975    /// Yields `before`, then stays pending until the gate is released, then
15976    /// yields `after` and reaches EOF. The bytes after the gate were written
15977    /// by a process that has already exited; only the reader is behind.
15978    struct HeldReader {
15979        before: Option<Vec<u8>>,
15980        gate: Option<oneshot::Receiver<()>>,
15981        after: io::Cursor<Vec<u8>>,
15982    }
15983
15984    impl AsyncRead for HeldReader {
15985        fn poll_read(
15986            mut self: Pin<&mut Self>,
15987            cx: &mut Context<'_>,
15988            buf: &mut ReadBuf<'_>,
15989        ) -> Poll<io::Result<()>> {
15990            if let Some(bytes) = self.before.take() {
15991                buf.put_slice(&bytes);
15992                return Poll::Ready(Ok(()));
15993            }
15994            if let Some(gate) = self.gate.as_mut() {
15995                match Pin::new(gate).poll(cx) {
15996                    Poll::Pending => return Poll::Pending,
15997                    Poll::Ready(_) => self.gate = None,
15998                }
15999            }
16000            Pin::new(&mut self.after).poll_read(cx, buf)
16001        }
16002    }
16003
16004    struct DiscardSink;
16005
16006    impl OutputSink for DiscardSink {
16007        fn write_line(&mut self, _line: &[u8]) {}
16008    }
16009
16010    fn line(text: &str) -> TailEntry {
16011        TailEntry::Line {
16012            text: text.to_string(),
16013            truncated: false,
16014            at_ms: None,
16015        }
16016    }
16017
16018    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16019        ring.lock().unwrap()
16020    }
16021
16022    /// Start a reader for a new process generation that delivers `before`
16023    /// immediately and `after` only once the returned sender fires (or is
16024    /// dropped).
16025    fn held_pump(
16026        ring: &Arc<Mutex<StderrRing>>,
16027        before: &str,
16028        after: &str,
16029    ) -> (StderrPump, oneshot::Sender<()>) {
16030        let generation = lock(ring).begin_process();
16031        let (release, gate) = oneshot::channel();
16032        let reader = HeldReader {
16033            before: Some(before.as_bytes().to_vec()),
16034            gate: Some(gate),
16035            after: io::Cursor::new(after.as_bytes().to_vec()),
16036        };
16037        let task = tokio::spawn(pump_stderr_to(
16038            reader,
16039            Arc::clone(ring),
16040            generation,
16041            DiscardSink,
16042        ));
16043        (StderrPump { task, generation }, release)
16044    }
16045
16046    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16047        for _ in 0..1000 {
16048            if done(&lock(ring)) {
16049                return;
16050            }
16051            tokio::time::sleep(Duration::from_millis(1)).await;
16052        }
16053        panic!(
16054            "ring never reached the expected state: {:?}",
16055            lock(ring).snapshot(None, None)
16056        );
16057    }
16058
16059    #[tokio::test(start_paused = true)]
16060    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16061        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16062        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16063
16064        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16065        let before_release = lock(&ring).snapshot(None, None);
16066        assert!(
16067            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16068            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16069        );
16070
16071        // The restart: the next process starts and writes before the old
16072        // reader catches up.
16073        let next = lock(&ring).begin_process();
16074        lock(&ring).push_line_from(next, "next process booting");
16075        release.send(()).unwrap();
16076        wait_until(&ring, |ring| {
16077            ring.snapshot(None, None).capture == CaptureState::Captured
16078        })
16079        .await;
16080
16081        assert_eq!(
16082            untimed(lock(&ring).snapshot(None, None).entries),
16083            vec![
16084                line("booting"),
16085                line("config error: missing storage"),
16086                TailEntry::ProcessStart,
16087                line("next process booting"),
16088            ],
16089            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16090        );
16091    }
16092
16093    #[tokio::test(start_paused = true)]
16094    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16095    ) {
16096        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16097        // `_held` is never fired: a descendant keeps the pipe open for the
16098        // whole test.
16099        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16100
16101        let started = Instant::now();
16102        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16103        assert_eq!(
16104            started.elapsed(),
16105            BOUND,
16106            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16107        );
16108
16109        let next = lock(&ring).begin_process();
16110        lock(&ring).push_line_from(next, "next process booting");
16111        tokio::time::sleep(Duration::from_secs(60)).await;
16112
16113        let snapshot = lock(&ring).snapshot(None, None);
16114        match &snapshot.capture {
16115            CaptureState::Incomplete { reason } => assert!(
16116                reason.contains("had not reached EOF") && reason.contains("250ms"),
16117                "the reason must say what is missing and after how long: {reason}"
16118            ),
16119            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16120        }
16121        assert_eq!(
16122            untimed(snapshot.entries),
16123            vec![
16124                line("parent exiting"),
16125                TailEntry::ProcessStart,
16126                line("next process booting"),
16127            ]
16128        );
16129    }
16130
16131    #[tokio::test(start_paused = true)]
16132    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16133        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16134        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16135        release.send(()).unwrap();
16136
16137        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16138
16139        let snapshot = lock(&ring).snapshot(None, None);
16140        assert_eq!(snapshot.capture, CaptureState::Captured);
16141        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16142    }
16143
16144    #[tokio::test(start_paused = true)]
16145    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16146        const BURST: usize = 150;
16147        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16148        let path = capture.join("two-pipe.stderr.log");
16149        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16150        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16151        let generation = lock(&ring).begin_process();
16152        let stdout = (0..BURST)
16153            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16154            .collect::<String>();
16155        let stderr = (0..BURST)
16156            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16157            .collect::<String>();
16158        // Model a child that has written both bursts and exited. Hold the stdout
16159        // reader after one line and a partial prefix, as if pipe readiness or
16160        // capture-file writes had delayed it. The final line also needs EOF to
16161        // flush: neither complete nor partial buffered output may be discarded.
16162        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16163        let (release, gate) = oneshot::channel();
16164        let reader = HeldReader {
16165            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16166            gate: Some(gate),
16167            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16168        };
16169        let (finished, completion) = oneshot::channel();
16170        let stdout_sink = sink.clone();
16171        let pump = tokio::spawn(async move {
16172            pump_stdout_to(reader, stdout_sink).await;
16173            finished.send(()).unwrap();
16174        });
16175        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16176
16177        let started = Instant::now();
16178        settle_stdout_pump("two-pipe", pump, BOUND).await;
16179        assert_eq!(
16180            started.elapsed(),
16181            BOUND,
16182            "a late reader must not hold up restart"
16183        );
16184        assert_eq!(
16185            std::fs::read_to_string(&path).unwrap().lines().count(),
16186            BURST + 1
16187        );
16188
16189        release
16190            .send(())
16191            .expect("stdout reader must survive the exit drain bound");
16192        tokio::time::timeout(Duration::from_secs(1), completion)
16193            .await
16194            .expect("released stdout reader must reach EOF")
16195            .expect("stdout capture must finish at EOF, not be cancelled");
16196        let contents = std::fs::read_to_string(&path).unwrap();
16197        let mut lines = contents
16198            .lines()
16199            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16200            .collect::<Vec<_>>();
16201        let mut expected = (0..BURST)
16202            .flat_map(|i| {
16203                [
16204                    format!("out-{i:04}-{}", "o".repeat(64)),
16205                    format!("err-{i:04}-{}", "e".repeat(64)),
16206                ]
16207            })
16208            .collect::<Vec<_>>();
16209        lines.sort_unstable();
16210        expected.sort_unstable();
16211        assert_eq!(
16212            lines, expected,
16213            "both pipes must retain every complete line exactly once"
16214        );
16215    }
16216}
16217
16218/// Containment of a module's process tree (issue #109).
16219///
16220/// The behaviour these defend against is a module helper surviving its module:
16221/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
16222/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
16223/// compounds it.
16224///
16225/// They run against the SUPERVISOR rather than the job-object crate because the
16226/// claim is about teardown: a crate-level test proves a job can reap a tree, not
16227/// that the daemon's drain path reaches it.
16228///
16229/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
16230/// lane there is a separate containment path with its own tests.
16231#[cfg(all(test, windows))]
16232mod job_containment_tests {
16233    use super::*;
16234    use cortexkit_test_support::ScratchDir;
16235    use std::{
16236        path::{Path, PathBuf},
16237        sync::{Arc, Mutex},
16238        time::{Duration, Instant},
16239    };
16240
16241    /// The stub, expected beside this test executable.
16242    ///
16243    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
16244    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
16245    /// failure then reads as a broken test rather than an unbuilt dependency.
16246    fn stub_path() -> PathBuf {
16247        let mut path = std::env::current_exe().expect("current_exe available in tests");
16248        path.pop();
16249        path.pop();
16250        path.push("fake-aft-stub.exe");
16251        assert!(
16252            path.exists(),
16253            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16254             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16255            path.display()
16256        );
16257        path
16258    }
16259
16260    /// Poll for the grandchild pid the stub records, and parse it.
16261    fn read_grandchild_pid(path: &Path) -> u32 {
16262        let deadline = Instant::now() + Duration::from_secs(10);
16263        loop {
16264            if let Ok(contents) = std::fs::read_to_string(path) {
16265                if let Ok(pid) = contents.trim().parse() {
16266                    return pid;
16267                }
16268            }
16269            assert!(
16270                Instant::now() < deadline,
16271                "the stub never recorded a grandchild pid at {}",
16272                path.display()
16273            );
16274            std::thread::sleep(Duration::from_millis(10));
16275        }
16276    }
16277
16278    /// Everything one fixture run needs, so the two tests below differ in exactly
16279    /// one place: whether the child is contained.
16280    struct Fixture {
16281        _dir: ScratchDir,
16282        module_id: String,
16283        grandchild: u32,
16284        child: Option<SupervisedChild>,
16285        registry: Arc<Registry>,
16286        snapshot: Arc<Mutex<SupervisorSnapshot>>,
16287        terminal_ring: Arc<Mutex<TerminalRing>>,
16288        spawn_events: SpawnEventFeed,
16289    }
16290
16291    fn fixture(label: &str, module_id: &str) -> Fixture {
16292        let dir = ScratchDir::new(label);
16293        let pid_file = dir.join("grandchild.pid");
16294        let supervisor = Supervisor::new_for_test(
16295            Arc::new(Registry::default()),
16296            RestartPolicy::new(3, Duration::ZERO),
16297        );
16298        let runtime = supervisor.runtime_config();
16299        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16300        let spec = ModuleSpec {
16301            module_id: module_id.to_string(),
16302            program: stub_path(),
16303            // Zero args deliberately: a `--subc` argument would make the stub dial
16304            // a daemon that is not there, and the failure would land in the same
16305            // stderr ring this fixture exists to keep quiet.
16306            args: Vec::new(),
16307            env: vec![
16308                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16309                (
16310                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16311                    pid_file.display().to_string(),
16312                ),
16313            ],
16314            reserved: false,
16315            reserved_prefixes: Vec::new(),
16316            protocol: ModuleProtocol::Subc,
16317            overlap: Default::default(),
16318        };
16319        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16320            .expect("spawn the supervised fixture");
16321        let grandchild = read_grandchild_pid(&pid_file);
16322        Fixture {
16323            _dir: dir,
16324            module_id: module_id.to_string(),
16325            grandchild,
16326            child: Some(child),
16327            registry: Arc::new(Registry::default()),
16328            snapshot,
16329            terminal_ring: Arc::clone(&runtime.terminal_ring),
16330            spawn_events: SpawnEventFeed::default(),
16331        }
16332    }
16333
16334    impl Fixture {
16335        /// Drain through the supervisor's own teardown path.
16336        async fn drain(&mut self) {
16337            let child = self
16338                .child
16339                .take()
16340                .expect("the fixture child is still present");
16341            drain_child_to_state(
16342                &self.module_id,
16343                ModuleProtocol::Subc,
16344                // No forwarding table in this fixture, so nothing reaches the
16345                // child over a connection.
16346                StopNotice::NotSent,
16347                &self.registry,
16348                None,
16349                &self.snapshot,
16350                &self.terminal_ring,
16351                &self.spawn_events,
16352                child,
16353                Duration::from_millis(500),
16354                ModuleState::Stopped,
16355                Some(false),
16356            )
16357            .await
16358            .expect("drain the supervised fixture");
16359        }
16360    }
16361
16362    /// Teardown reaps the grandchild, not merely the direct child.
16363    ///
16364    /// This is the assertion the change exists for. Before containment the
16365    /// grandchild survived: it is a separate process, and `start_kill` is
16366    /// `TerminateProcess` scoped to one pid.
16367    #[tokio::test]
16368    async fn teardown_reaps_the_grandchild() {
16369        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16370        let grandchild = fixture.grandchild;
16371
16372        assert!(
16373            subc_jobobject::process_exists(grandchild),
16374            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16375        );
16376
16377        fixture.drain().await;
16378
16379        assert!(
16380            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16381            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16382        );
16383    }
16384
16385    /// The mutation control: with containment withheld, the grandchild survives
16386    /// the same kill.
16387    ///
16388    /// This is the defect reproduction from #109 — a direct-child kill reaches
16389    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16390    /// supervisor because `spawn_and_mark_running` now always contains on
16391    /// Windows, which is the point: there is no longer a path that spawns
16392    /// uncontained, so the control has to construct one.
16393    ///
16394    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16395    /// grandchild ever dies here, that test is passing for a reason unrelated to
16396    /// the job object and the containment claim is unproven.
16397    #[test]
16398    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16399        let dir = ScratchDir::new("teardown-uncontained");
16400        let pid_file = dir.join("grandchild.pid");
16401        let mut child = std::process::Command::new(stub_path())
16402            .env("FAKE_AFT_NEVER_CONNECT", "1")
16403            .env(
16404                "FAKE_AFT_GRANDCHILD_PID_FILE",
16405                pid_file.display().to_string(),
16406            )
16407            .stdin(std::process::Stdio::null())
16408            .stdout(std::process::Stdio::null())
16409            .stderr(std::process::Stdio::null())
16410            .spawn()
16411            .expect("spawn the uncontained fixture");
16412        let grandchild = read_grandchild_pid(&pid_file);
16413
16414        // Exactly what the pre-fix teardown did: kill the direct child.
16415        child.kill().expect("kill the direct child");
16416        let _ = child.wait();
16417
16418        assert!(
16419            subc_jobobject::process_exists(grandchild),
16420            "grandchild {grandchild} died with the direct child, so this control no longer \
16421             distinguishes contained from uncontained teardown and the regression test is \
16422             passing vacuously"
16423        );
16424
16425        // The orphan this control demonstrates is the leak the fix prevents, so
16426        // the control must not leave one behind.
16427        kill_tree(grandchild);
16428    }
16429
16430    /// Crash durability: closing the containment handle reaps the tree with no
16431    /// teardown code running at all.
16432    ///
16433    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16434    /// call anything — and it is why containment is a kernel property of the
16435    /// handle rather than a step in the drain. Discovered by getting the
16436    /// mutation control wrong: clearing `job` to "disable" containment instead
16437    /// killed the tree, which is the guarantee, not a mistake.
16438    #[tokio::test]
16439    async fn dropping_containment_reaps_the_grandchild() {
16440        let mut fixture = fixture("drop-containment", "tree-drop");
16441        let grandchild = fixture.grandchild;
16442
16443        assert!(subc_jobobject::process_exists(grandchild));
16444
16445        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16446        fixture.child.as_mut().expect("child present").job = None;
16447
16448        assert!(
16449            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16450            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16451             crash would leave the tree behind"
16452        );
16453    }
16454
16455    /// Kill a pid and its tree, then confirm it is gone.
16456    fn kill_tree(pid: u32) {
16457        let _ = std::process::Command::new("taskkill.exe")
16458            .args(["/PID", &pid.to_string(), "/T", "/F"])
16459            .stdin(std::process::Stdio::null())
16460            .stdout(std::process::Stdio::null())
16461            .stderr(std::process::Stdio::null())
16462            .status();
16463        assert!(
16464            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16465            "could not clean up grandchild {pid}"
16466        );
16467    }
16468}
16469
16470#[cfg(test)]
16471mod privacy_trampoline_configuration_tests {
16472    #[tokio::test]
16473    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16474    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16475        #[cfg(target_os = "macos")]
16476        {
16477            let supervisor = super::Supervisor::new(
16478                std::sync::Arc::new(crate::Registry::default()),
16479                super::RestartPolicy::default(),
16480            );
16481            let error = supervisor.spawn(spec()).unwrap_err();
16482            assert!(
16483                error
16484                    .to_string()
16485                    .contains("no privacy trampoline configured"),
16486                "{error}"
16487            );
16488        }
16489    }
16490
16491    #[tokio::test]
16492    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16493    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16494        #[cfg(target_os = "macos")]
16495        {
16496            let supervisor = super::Supervisor::new(
16497                std::sync::Arc::new(crate::Registry::default()),
16498                super::RestartPolicy::default(),
16499            )
16500            .with_privacy_trampoline(std::env::current_exe().unwrap());
16501            let error = supervisor.spawn(spec()).unwrap_err();
16502            assert!(
16503                error
16504                    .to_string()
16505                    .contains("binary does not implement the privacy trampoline protocol"),
16506                "{error}"
16507            );
16508        }
16509    }
16510
16511    #[cfg(target_os = "macos")]
16512    fn spec() -> super::ModuleSpec {
16513        super::ModuleSpec {
16514            module_id: "privacy-configuration".into(),
16515            program: "/bin/sleep".into(),
16516            args: vec!["30".into()],
16517            env: vec![],
16518            reserved: false,
16519            reserved_prefixes: vec![],
16520            protocol: subc_control::ModuleProtocol::None,
16521            overlap: super::ModuleOverlap::Exclusive,
16522        }
16523    }
16524}
16525
16526#[cfg(test)]
16527mod privacy_exec_boundary_tests {
16528    #[cfg(target_os = "macos")]
16529    use super::*;
16530    #[cfg(target_os = "macos")]
16531    use std::{
16532        io::{Read, Write},
16533        net::{TcpListener, TcpStream},
16534    };
16535
16536    /// Unit-test-only pause at the actual early image read, not at a later
16537    /// status read. Production supervisors never inspect this environment key.
16538    #[cfg(target_os = "macos")]
16539    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16540        if let Some((_, path)) = spec
16541            .env
16542            .iter()
16543            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16544        {
16545            let mut barrier = TcpStream::connect(path).unwrap();
16546            barrier
16547                .set_read_timeout(Some(Duration::from_secs(30)))
16548                .unwrap();
16549            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16550            let mut release = [0];
16551            barrier.read_exact(&mut release).unwrap();
16552            assert_eq!(&release, b"X");
16553        }
16554    }
16555
16556    #[cfg(target_os = "macos")]
16557    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16558        ModuleSpec {
16559            module_id: "privacy-boundary".into(),
16560            program: program.into(),
16561            args: args.iter().map(|arg| (*arg).into()).collect(),
16562            env: vec![],
16563            reserved: false,
16564            reserved_prefixes: vec![],
16565            protocol: ModuleProtocol::None,
16566            overlap: ModuleOverlap::Exclusive,
16567        }
16568    }
16569
16570    #[cfg(target_os = "macos")]
16571    async fn accept(listener: TcpListener) -> TcpStream {
16572        // Socket readiness, not elapsed time, establishes both pause points.
16573        let listener = tokio::net::TcpListener::from_std({
16574            listener.set_nonblocking(true).unwrap();
16575            listener
16576        })
16577        .unwrap();
16578        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16579            .await
16580            .unwrap()
16581            .unwrap();
16582        let stream = stream.into_std().unwrap();
16583        stream.set_nonblocking(false).unwrap();
16584        stream
16585            .set_read_timeout(Some(Duration::from_secs(30)))
16586            .unwrap();
16587        stream
16588    }
16589
16590    #[tokio::test]
16591    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16592    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16593        #[cfg(target_os = "macos")]
16594        {
16595            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16596            // Loopback sockets also work when the replay adapter's TMPDIR is
16597            // longer than Darwin's Unix-domain socket path limit.
16598            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16599            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16600            let record = root.join("live-children.json");
16601            let supervisor =
16602                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16603                    .with_live_children_record(&record);
16604            let runtime = supervisor.runtime_config();
16605            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16606            let mut spec = spec("/bin/sleep", &["30"]);
16607            spec.env = vec![
16608                (
16609                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16610                    exec_listener.local_addr().unwrap().to_string(),
16611                ),
16612                (
16613                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16614                    sample_listener.local_addr().unwrap().to_string(),
16615                ),
16616            ];
16617            let spawn = tokio::task::spawn_blocking(move || {
16618                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16619            });
16620            let mut sample = accept(sample_listener).await;
16621            let mut pid = [0; 4];
16622            sample.read_exact(&mut pid).unwrap();
16623            let pid = u32::from_ne_bytes(pid);
16624            let mut exec = accept(exec_listener).await;
16625            let mut ready = [0];
16626            exec.read_exact(&mut ready).unwrap();
16627            assert_eq!(&ready, b"R");
16628            // The early read is guaranteed to see a real, non-null trampoline
16629            // image: the fixture has reached its barrier and cannot exec yet.
16630            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16631            assert_eq!(
16632                observe_spawned_image(pid).unwrap().executable,
16633                Some(trampoline)
16634            );
16635            sample.write_all(b"X").unwrap();
16636            let mut child = spawn.await.unwrap();
16637            let early = crate::live_children::read_record(&record).unwrap();
16638            assert_eq!(early.len(), 1);
16639            assert_eq!(early[0].pid, pid);
16640            assert_eq!(
16641                early[0].executable, None,
16642                "unconfirmed trampoline image entered the roster"
16643            );
16644            assert!(child.report_ready.get().is_none());
16645            // The barrier's duration is unrelated to the production five-second
16646            // exec budget. Start the test's confirmation budget upon release.
16647            child.privacy_exec.as_mut().unwrap().deadline =
16648                tokio::time::Instant::now() + Duration::from_secs(30);
16649            exec.write_all(b"X").unwrap();
16650            child.confirm_privacy_exec().await;
16651            assert_eq!(child.spawn_failure, None);
16652            assert!(child.report_ready.get().is_some());
16653            let confirmed = crate::live_children::read_record(&record).unwrap();
16654            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16655            assert_ne!(module, trampoline);
16656            assert_eq!(confirmed[0].executable, Some(module.into()));
16657            child.start_kill().unwrap();
16658            child.wait().await.unwrap();
16659            child.release_roster();
16660        }
16661    }
16662
16663    #[tokio::test]
16664    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16665    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16666        #[cfg(target_os = "macos")]
16667        {
16668            let registry = Arc::new(Registry::default());
16669            let policy = RestartPolicy::new(0, Duration::ZERO);
16670            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16671            let runtime = supervisor.runtime_config();
16672            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16673            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16674            // Drive spawn and confirmation separately instead of starting the
16675            // monitor. WNOWAIT observes a real exit without consuming its status,
16676            // so confirmation's first try_wait must take the already-exited arm.
16677            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16678            let pid = child.pid;
16679            tokio::task::spawn_blocking(move || {
16680                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16681            })
16682            .await
16683            .unwrap()
16684            .unwrap();
16685            child.privacy_exec.as_mut().unwrap().deadline =
16686                tokio::time::Instant::now() + Duration::from_secs(30);
16687            let status = child.wait().await.unwrap();
16688            assert_eq!(status.code(), Some(121));
16689            assert!(child.privacy_exec.is_none());
16690            assert!(
16691                child.report_ready.get().is_none(),
16692                "an exited module must not publish a live pid"
16693            );
16694            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16695            on_child_exit(
16696                &spec,
16697                policy,
16698                &registry,
16699                &snapshot,
16700                &runtime.terminal_ring,
16701                &runtime.spawn_events,
16702                &runtime.child_roster,
16703                report,
16704            )
16705            .await;
16706            let state = lock_snapshot(&snapshot).unwrap();
16707            assert_eq!(state.state, ModuleState::Failed);
16708            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16709            assert_eq!(state.reported_pid(), None);
16710            drop(state);
16711            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16712            assert_eq!(history.entries.len(), 1);
16713            let terminal = &history.entries[0];
16714            assert_eq!(terminal.exit_code, Some(121));
16715            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16716            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16717            assert_eq!(
16718                terminal.disposition_detail.as_deref(),
16719                Some(policy.budget_exhausted_detail().as_str()),
16720                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16721            );
16722            assert_eq!(child.spawn_failure, None);
16723            child.release_roster();
16724        }
16725    }
16726}
16727
16728/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16729/// descriptor 3, without an environment copy. The shell records the nonce
16730/// and its environment after exec so these tests observe the real handover.
16731#[cfg(all(test, unix))]
16732mod launch_nonce_descriptor_tests {
16733    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16734    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16735    use cortexkit_test_support::ScratchDir;
16736    use std::{
16737        path::PathBuf,
16738        sync::{Arc, Mutex},
16739        time::{Duration, Instant},
16740    };
16741
16742    async fn probe(role: super::SpawnRole) {
16743        let scratch = ScratchDir::new("launch-nonce-descriptor");
16744        let fd_copy = scratch.join("from-descriptor");
16745        let env_copy = scratch.join("environment");
16746        let script = format!(
16747            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16748            fd = fd_copy.display(), env = env_copy.display(),
16749        );
16750        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16751        let spec = ModuleSpec {
16752            module_id: "nonce-descriptor-probe".to_string(),
16753            program: PathBuf::from("/bin/sh"),
16754            args: vec!["-c".to_string(), script],
16755            env: vec![
16756                xdg("XDG_DATA_HOME"),
16757                xdg("XDG_RUNTIME_DIR"),
16758                xdg("XDG_CONFIG_HOME"),
16759                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16760            ],
16761            reserved: true,
16762            reserved_prefixes: Vec::new(),
16763            protocol: ModuleProtocol::Subc,
16764            overlap: Default::default(),
16765        };
16766        let handle = SupervisorHandle::new();
16767        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16768        let roster = ChildRoster::default();
16769        #[cfg(target_os = "macos")]
16770        {
16771            let path = super::test_privacy_trampoline();
16772            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16773        }
16774        let child = super::spawn_child_in_slot(
16775            &spec,
16776            None,
16777            Some(&handle),
16778            &ring,
16779            None,
16780            &roster,
16781            #[cfg(target_os = "linux")]
16782            None,
16783            role,
16784            matches!(role, super::SpawnRole::SwapCandidate),
16785        )
16786        .expect("spawn probe");
16787        let deadline = Instant::now() + Duration::from_secs(10);
16788        while !(fd_copy.exists() && env_copy.exists()) {
16789            assert!(Instant::now() < deadline, "probe never wrote its copies");
16790            tokio::time::sleep(Duration::from_millis(20)).await;
16791        }
16792        let nonce = std::fs::read_to_string(fd_copy).unwrap();
16793        assert!(!nonce.is_empty());
16794        let environment = std::fs::read_to_string(env_copy).unwrap();
16795        assert!(environment
16796            .lines()
16797            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16798        let copy = environment
16799            .lines()
16800            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16801        assert_eq!(
16802            copy, None,
16803            "Unix children must never receive the environment nonce"
16804        );
16805        if matches!(role, super::SpawnRole::Plain) {
16806            assert_eq!(
16807                handle.spawn_nonce(&spec.module_id).as_deref(),
16808                Some(nonce.as_str())
16809            );
16810        }
16811        drop(child);
16812    }
16813
16814    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16815    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16816        probe(super::SpawnRole::Plain).await;
16817    }
16818
16819    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16820    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16821        probe(super::SpawnRole::SwapCandidate).await;
16822    }
16823}
16824
16825#[cfg(all(test, target_os = "linux"))]
16826mod cgroup_containment_tests {
16827    use super::*;
16828    use cortexkit_test_support::ScratchDir;
16829
16830    fn running(pid: u32) -> bool {
16831        // An orphan can remain a zombie until the container init reaps it.
16832        std::fs::read_to_string(format!("/proc/{pid}/stat"))
16833            .ok()
16834            .and_then(|stat| {
16835                stat.rsplit_once(") ")
16836                    .map(|(_, rest)| rest.starts_with('Z'))
16837            })
16838            .is_some_and(|zombie| !zombie)
16839    }
16840
16841    #[tokio::test]
16842    async fn linux_teardown_reaps_the_grandchild() {
16843        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16844    }
16845
16846    #[tokio::test]
16847    async fn linux_shutdown_straggler_reaps_the_grandchild() {
16848        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16849    }
16850
16851    async fn teardown_tree(test_name: &str, shutdown: bool) {
16852        let dir = ScratchDir::new(test_name);
16853        let root = PathBuf::from(format!(
16854            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16855            std::process::id(),
16856            unix_ms_now()
16857        ));
16858        if let Err(error) = std::fs::create_dir(&root) {
16859            assert!(
16860                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16861                "required cgroup test cannot execute: {error}"
16862            );
16863            eprintln!(
16864                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16865                root.display()
16866            );
16867            return;
16868        }
16869        let placement = subc_cgroup::prepare_at(&root)
16870            .expect("prepare isolated kernel cgroup")
16871            .expect("isolated cgroup is delegated");
16872        let module_id = "tree-teardown";
16873        let module = placement
16874            .module_path(module_id)
16875            .expect("create isolated module cgroup");
16876        if !module.join("cgroup.kill").exists() {
16877            std::fs::remove_dir(&module).unwrap();
16878            std::fs::remove_dir(root.join("subc-modules")).unwrap();
16879            std::fs::remove_dir(&root).unwrap();
16880            assert!(
16881                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16882                "required cgroup.kill interface unavailable"
16883            );
16884            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16885            return;
16886        }
16887        let supervisor = Supervisor::new_for_test(
16888            Arc::new(Registry::default()),
16889            RestartPolicy::new(3, Duration::ZERO),
16890        )
16891        .with_cgroup_placement(Some(placement));
16892        let mut runtime = supervisor.runtime_config();
16893        runtime.child_roster = runtime
16894            .child_roster
16895            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16896        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16897        let pid_file = dir.join("grandchild.pid");
16898        let spec = ModuleSpec {
16899            module_id: module_id.to_string(),
16900            program: PathBuf::from("/bin/sh"),
16901            args: vec![
16902                "-c".into(),
16903                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16904                "fixture".into(),
16905                pid_file.display().to_string(),
16906            ],
16907            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16908                .into_iter()
16909                .map(|key| (key.to_string(), dir.display().to_string()))
16910                .collect(),
16911            reserved: false,
16912            reserved_prefixes: Vec::new(),
16913            protocol: ModuleProtocol::None,
16914            overlap: Default::default(),
16915        };
16916        let child =
16917            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16918        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16919        let grandchild: u32 = loop {
16920            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16921                if let Ok(pid) = contents.trim().parse() {
16922                    break pid;
16923                }
16924            }
16925            assert!(
16926                tokio::time::Instant::now() < deadline,
16927                "grandchild pid was not recorded"
16928            );
16929            tokio::time::sleep(Duration::from_millis(10)).await;
16930        };
16931        assert!(
16932            running(grandchild),
16933            "grandchild must be alive before teardown"
16934        );
16935        if shutdown {
16936            let mut child = child;
16937            crate::child_roster::end_children_for_daemon_shutdown(
16938                &runtime.child_roster,
16939                false,
16940                std::future::pending(),
16941            )
16942            .await;
16943            child.wait().await.expect("reap shutdown straggler");
16944        } else {
16945            drain_child_to_state(
16946                module_id,
16947                ModuleProtocol::None,
16948                StopNotice::NotSent,
16949                &Registry::default(),
16950                None,
16951                &snapshot,
16952                &runtime.terminal_ring,
16953                &SpawnEventFeed::default(),
16954                child,
16955                Duration::from_millis(100),
16956                ModuleState::Stopped,
16957                Some(false),
16958            )
16959            .await
16960            .expect("real supervisor teardown");
16961        }
16962        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16963        while running(grandchild) && tokio::time::Instant::now() < deadline {
16964            tokio::time::sleep(Duration::from_millis(10)).await;
16965        }
16966        let survived = running(grandchild);
16967        // Kill a surviving grandchild so a failed test does not leave it behind.
16968        if survived {
16969            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16970            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16971            tokio::time::sleep(Duration::from_millis(100)).await;
16972        }
16973        if module.exists() {
16974            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16975        }
16976        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16977        std::fs::remove_dir(&root).unwrap();
16978        assert!(
16979            !survived,
16980            "grandchild {grandchild} outlived module teardown"
16981        );
16982        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16983    }
16984}
16985
16986/// A slow live-children record write on the restart path must not stop the
16987/// daemon from answering the replacement module's authentication handshake.
16988#[cfg(all(test, unix))]
16989mod record_write_stall_tests {
16990    use super::*;
16991    use std::sync::atomic::{AtomicBool, Ordering};
16992    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16993
16994    /// `subc-client-rs` gives the whole handshake this long before the module
16995    /// exits with `Timeout { stage: ServerProof }`.
16996    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16997    /// How long the replacement's admission write is held: longer than the
16998    /// module's deadline, standing in for the delay a busy disk can add
16999    /// across the two syncs in that write.
17000    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17001
17002    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
17003    /// On macOS every module is launched through the privacy trampoline, and
17004    /// a module that is not a `#!` script is admitted only if its running
17005    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
17006    /// the shell soon replaces itself with `/bin/sleep`, which does not match
17007    /// the configured `/bin/sh`. The image is read only after the spawn's
17008    /// live-children record write, and here that write lasted long enough for
17009    /// the shell to have exec'd every time, so each process was refused and
17010    /// killed right after spawn, and the supervisor crash-looped it with
17011    /// backoff until its restart budget ran out. The restart this test
17012    /// measures would then never produce a module at all.
17013    fn sleeping_spec() -> ModuleSpec {
17014        ModuleSpec {
17015            module_id: "record-write-stall".into(),
17016            program: "/bin/sleep".into(),
17017            args: vec!["600".into()],
17018            env: vec![],
17019            reserved: false,
17020            reserved_prefixes: vec![],
17021            protocol: ModuleProtocol::None,
17022            overlap: Default::default(),
17023        }
17024    }
17025
17026    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17027    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17028        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17029        let supervisor =
17030            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17031                .with_live_children_record(home.join("live-children.json"));
17032        let module = supervisor.spawn(sleeping_spec()).unwrap();
17033        // `reported_pid` names a process only once it is confirmed as the
17034        // module (on macOS, once the trampoline's image check has passed), so
17035        // a process the daemon is about to refuse and kill never counts as
17036        // the running module, here or in the final wait below.
17037        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17038            loop {
17039                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17040                    .unwrap()
17041                    .reported_pid()
17042                {
17043                    return pid;
17044                }
17045                tokio::time::sleep(Duration::from_millis(10)).await;
17046            }
17047        })
17048        .await
17049        .expect("the first process was admitted as the module");
17050
17051        // The daemon's listener, served by this runtime's workers like the
17052        // supervisor's own tasks.
17053        let key = vec![0x42; 32];
17054        let daemon_id = [0x24; 16];
17055        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17056        let port = listener.local_addr().unwrap().port();
17057        let server = tokio::spawn(crate::server::serve_listener(
17058            listener,
17059            Arc::new(crate::router::Router::with_default_self_handler()),
17060            crate::server::ServerAuth::with_limits(
17061                key.clone(),
17062                daemon_id,
17063                "test",
17064                crate::server::DEFAULT_AUTH_DEADLINE,
17065                4,
17066            ),
17067        ));
17068
17069        // Hold the write that admits the replacement, once, and say when it
17070        // has started.
17071        let (held_tx, held_rx) = std::sync::mpsc::channel();
17072        let armed = AtomicBool::new(true);
17073        supervisor
17074            .child_roster
17075            .set_record_write_hook_for_test(move |live| {
17076                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17077                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17078                    let _ = held_tx.send(());
17079                    std::thread::sleep(HELD_RECORD_WRITE);
17080                }
17081            });
17082
17083        // The replacement module, as a separate process would be: its own
17084        // thread and runtime, so only the daemon's runtime can be stalled.
17085        let module_process = std::thread::spawn(move || {
17086            held_rx
17087                .recv_timeout(Duration::from_secs(30))
17088                .expect("the restart reached the replacement's record write");
17089            let conn = ConnectionInfo {
17090                schema: SCHEMA_VERSION,
17091                wire_version: None,
17092                endpoints: vec![Endpoint {
17093                    host: "127.0.0.1".to_owned(),
17094                    port,
17095                }],
17096                key,
17097                daemon_id,
17098                pid: std::process::id(),
17099                daemon_ver: "test".to_owned(),
17100            };
17101            tokio::runtime::Builder::new_current_thread()
17102                .enable_all()
17103                .build()
17104                .unwrap()
17105                .block_on(async {
17106                    let started = Instant::now();
17107                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17108                        .await
17109                        .unwrap();
17110                    let result =
17111                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17112                    (result, started.elapsed())
17113                })
17114        });
17115
17116        module.restart(Some(0)).await.unwrap();
17117        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17118            .await
17119            .unwrap();
17120        assert!(
17121            auth.is_ok(),
17122            "the daemon must answer a module's handshake while a restart waits on the \
17123             live-children record write; got {auth:?} after {elapsed:?}"
17124        );
17125
17126        tokio::time::timeout(Duration::from_secs(20), async {
17127            while lock_snapshot(&module.inner.snapshot)
17128                .unwrap()
17129                .reported_pid()
17130                .is_none_or(|pid| pid == old_pid)
17131            {
17132                tokio::time::sleep(Duration::from_millis(10)).await;
17133            }
17134        })
17135        .await
17136        .expect("the restart published its replacement");
17137        module.stop().await.unwrap();
17138        server.abort();
17139    }
17140}