Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68/// Command-line flag used by supervised modules to find subc.
69///
70/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
71/// The path points at the TCP+key connection file; it is not an ambient signal and
72/// is never inherited by standalone children.
73pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
79/// to contain a real crash loop (which respawns in seconds) and short enough
80/// that unrelated crashes hours apart never accumulate into a permanent stop.
81const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82/// How long a drain waits for already-dispatched requests to finalize before
83/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
84/// not RPC-scale: the original 2s value silently cut nearly every real tool
85/// call at the fence, making the wait-for-finalize design decorative for the
86/// workloads it existed for. Quiescence short-circuits, so an idle module
87/// restarts immediately regardless of this value; the budget is spent only
88/// when a genuine in-flight request is worth finishing. Per-module override:
89/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
90/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
91/// where a stuck request will never settle).
92pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
94const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
95/// How long a drain waits after one busy-gauge probe completes before it asks
96/// again.
97///
98/// A drain re-probes on this short interval of its own rather than on the
99/// module's health cadence. The default cadence (30 s) equals the default
100/// drain ceiling, so a cadence-paced drain that found the module busy at its
101/// first probe never asked a second time: it waited out the whole ceiling and
102/// forced GOODBYE even after the gauges had read zero. The interval is counted
103/// from the moment the previous probe finished, and probes run one at a time,
104/// so a slow module is never asked more often than it can answer. Periodic
105/// health checks outside a drain keep using the configured cadence.
106const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
107/// How long a restart waits for the exited process's output readers.
108///
109/// The restart does not depend on either output reader finishing. A stderr
110/// reader still running at this bound is left running, and its later output goes
111/// into the exited process's own section of the stderr ring (see
112/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
113/// bound no longer decides whether a crash's last lines are kept: under load
114/// the reader may simply not have been scheduled yet, and cutting it there
115/// discarded exactly the lines that explained the crash.
116///
117/// What the bound still decides is when the tail starts reporting
118/// `Incomplete`: a pipe open past it usually means a descendant of the exited
119/// process still holds it, and the tail cannot claim to be whole until that
120/// pipe closes. That is also the only case in which the wait costs the restart
121/// anything, because a pipe with no other holder reaches EOF when the process
122/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
123/// `Captured` at EOF with nothing lost.
124///
125/// The stdout reader carries no ring, only the capture file. It also continues
126/// to EOF after this bound: child exit closes the writer but does not mean the
127/// reader has consumed all buffered bytes. Late output from either pipe can
128/// therefore reach the shared capture file after the successor starts.
129const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
130/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
131pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
132const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
133/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
134/// dropped because the subscriber stopped draining its frames. The detail's
135/// `first_undelivered_cursor` names the first event it did not receive; the
136/// client resubscribes from the last cursor it did receive.
137pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
138
139#[cfg(target_os = "macos")]
140struct PrivacyExec {
141    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
142    deadline: tokio::time::Instant,
143    expected: Option<subc_os::FileIdentity>,
144    trampoline: Option<subc_os::FileIdentity>,
145    script: bool,
146    module_id: String,
147    #[cfg(test)]
148    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
149}
150
151#[cfg(all(test, target_os = "macos"))]
152pub(crate) fn test_privacy_trampoline() -> PathBuf {
153    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
154    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
155    let path = std::env::current_exe()
156        .unwrap()
157        .parent()
158        .unwrap()
159        .parent()
160        .unwrap()
161        .join("privacy-trampoline-fixture");
162    // Without the fixture every macOS spawn is refused, and the tests that
163    // spawn fail later as a module in state Failed, which names the wrong
164    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
165    assert!(
166        path.exists(),
167        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
168         --bins --features test-support` or `cargo test -p subc-daemon` first",
169        path.display()
170    );
171    path
172}
173
174#[cfg(target_os = "macos")]
175fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
176    use std::io::Read;
177    let mut probe = std::process::Command::new(path)
178        .args(["__disclaim-exec", "--probe"])
179        .stdin(Stdio::null())
180        .stdout(Stdio::piped())
181        .stderr(Stdio::piped())
182        .spawn()
183        .map_err(|error| {
184            format!(
185                "privacy trampoline probe failed for {}: {error}",
186                path.display()
187            )
188        })?;
189    let deadline = std::time::Instant::now() + Duration::from_secs(5);
190    let status = loop {
191        match probe.try_wait() {
192            Ok(Some(status)) => break status,
193            Ok(None) if std::time::Instant::now() < deadline => {
194                std::thread::sleep(Duration::from_millis(5))
195            }
196            result => {
197                let _ = probe.kill();
198                let _ = probe.wait();
199                return Err(format!(
200                    "privacy trampoline probe failed or timed out for {}: {result:?}",
201                    path.display()
202                ));
203            }
204        }
205    };
206    let mut answer = String::new();
207    if let Some(stdout) = probe.stdout.take() {
208        let _ = stdout.take(256).read_to_string(&mut answer);
209    }
210    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
211        return Ok(());
212    }
213    let mut diagnostic = String::new();
214    if let Some(stderr) = probe.stderr.take() {
215        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
216    }
217    let cause = diagnostic
218        .trim()
219        .strip_prefix("ck-subc: own privacy identity refused: ")
220        .unwrap_or("binary does not implement the privacy trampoline protocol");
221    Err(format!(
222        "{cause}: probe of {} exited {status}",
223        path.display()
224    ))
225}
226
227#[cfg(target_os = "macos")]
228fn privacy_command(
229    spec: &ModuleSpec,
230    roster: &ChildRoster,
231) -> Result<
232    (
233        Command,
234        Option<PrivacyExec>,
235        subc_os::privacy_identity::ExecAcknowledgement,
236    ),
237    SuperviseError,
238> {
239    let failure = |cause: String| {
240        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
241        SuperviseError::Spawn {
242            program: spec.program.clone(),
243            source: io::Error::other(cause),
244            cgroup_path: None,
245        }
246    };
247    let trampoline = roster.privacy_trampoline().map_err(failure)?;
248    // Resolve PATH with the same environment the Command will receive. For
249    // scripts retain the existing orphan-identity rule: the kernel chooses
250    // the interpreter, and its observed image is the one recorded. Do not
251    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
252    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
253        let path = spec
254            .env
255            .iter()
256            .find(|(key, _)| key == "PATH")
257            .map(|(_, value)| std::ffi::OsString::from(value))
258            .or_else(|| std::env::var_os("PATH"))
259            .unwrap_or_else(|| "/usr/bin:/bin".into());
260        std::env::split_paths(&path)
261            .map(|dir| dir.join(&spec.program))
262            .find(|path| path.is_file())
263            .unwrap_or_else(|| spec.program.clone())
264    } else {
265        spec.program.clone()
266    };
267    let expected = subc_os::file_identity(&program);
268    let trampoline_image = subc_os::file_identity(&trampoline);
269    let script = {
270        use std::io::Read;
271        let mut prefix = [0u8; 2];
272        std::fs::File::open(&program)
273            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
274    };
275    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
276        return Err(failure(
277            "privacy identity module executable is missing or is the trampoline itself".to_string(),
278        ));
279    }
280    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
281        .map_err(|error| failure(error.to_string()))?;
282    let reader =
283        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
284    let mut command = Command::new(&trampoline);
285    command
286        .arg("__disclaim-exec")
287        .arg(ack.fd().to_string())
288        .arg(&program);
289    ack.install(command.as_std_mut());
290    Ok((
291        command,
292        Some(PrivacyExec {
293            reader,
294            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
295            expected,
296            trampoline: trampoline_image,
297            script,
298            module_id: spec.module_id.clone(),
299            #[cfg(test)]
300            confirmation_pause: spec
301                .env
302                .iter()
303                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
304                .map(|(_, address)| (address.clone(), None)),
305        }),
306        ack,
307    ))
308}
309
310struct SupervisedChild {
311    child: Child,
312    exec: crate::activity::ExecOwner,
313    #[cfg(windows)]
314    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
315    #[cfg(target_os = "macos")]
316    privacy_exec: Option<PrivacyExec>,
317    /// Set once this launch's exec acknowledgement confirms the module image.
318    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
319    /// `subc_os::privacy_identity`), which then replaces itself with the
320    /// module. The supervisor owns and can kill that pid from spawn, but
321    /// status readers report it only after this latch is set, so nothing
322    /// reports the trampoline's image as the module's.
323    #[cfg(target_os = "macos")]
324    report_ready: Arc<OnceLock<()>>,
325    /// Refusal before the module image was accepted, retained for terminal records.
326    spawn_failure: Option<String>,
327    /// The protocol this process was launched with. A reload can store a new
328    /// launch spec with a different protocol, but that takes effect only at the
329    /// next spawn, so this process keeps being handled by the protocol it
330    /// actually speaks.
331    protocol: ModuleProtocol,
332    /// This process's cgroup name: a bounded module/slot label followed by a
333    /// spawn suffix unique to this process (when cgroup placement is on). A
334    /// retired process in a slot may still be draining when a later one is
335    /// spawned into that slot, so the suffix keeps the later process out of
336    /// the retired one's cgroup, which is the domain a kill applies to.
337    #[cfg(target_os = "linux")]
338    module_id: String,
339    #[cfg(target_os = "linux")]
340    cgroup_placement: Option<subc_cgroup::Placement>,
341    /// The job that contains this child and every process it spawns (issue #109).
342    ///
343    /// Dropping this handle is what reaps a surviving tree when no supervisor
344    /// code runs — a daemon crash — because the job carries
345    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
346    ///
347    /// That limit is not crash-only, and the difference is worth knowing: a
348    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
349    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
350    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
351    /// module at once. Before this change they survived that, saw EOF on the
352    /// control socket, and ran their own teardown; Unix keeps that path
353    /// deliberately, so a module can seal a WAL or close a capture rather than
354    /// be killed mid-write. So this trades graceful teardown on every Windows
355    /// daemon stop for containment on a crash, which is the right way round
356    /// today: orphaned GPU workers are a reported, recurring problem, and the
357    /// modules that write most heavily do not run on Windows.
358    ///
359    /// The fix is a real Windows stop path — the daemon draining before it
360    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
361    /// reaches only what the drain left behind, which is what it should reach.
362    #[cfg(windows)]
363    job: Option<subc_jobobject::JobObject>,
364    stdout_pump: Option<JoinHandle<()>>,
365    stderr_pump: Option<StderrPump>,
366    stderr_ring: Arc<Mutex<StderrRing>>,
367    /// Conservative monotonic bound sampled before `command.spawn()`, not when
368    /// identity recording finishes or the spawn generation is published.
369    spawned_at: Instant,
370    spawned_at_ms: u64,
371    spawned_from: PathBuf,
372    launch_spec_digest: LaunchSpecDigest,
373    spawned_file_identity: Option<SpawnedFileIdentity>,
374    process_start_time: Option<u64>,
375    process_identity: Option<ProcessIdentity>,
376    pid: u32,
377    /// This process's entry in the daemon's child roster, released when the
378    /// process is reaped or this handle is dropped.
379    roster_guard: Option<crate::child_roster::RosterGuard>,
380}
381
382impl SupervisedChild {
383    fn id(&self) -> Option<u32> {
384        Some(self.pid)
385    }
386
387    fn process_identity(&self) -> Option<ProcessIdentity> {
388        self.process_identity
389    }
390
391    async fn wait(&mut self) -> io::Result<ExitStatus> {
392        #[cfg(target_os = "macos")]
393        self.confirm_privacy_exec().await;
394        // The roster entry is NOT released here. A daemon shutdown waits for the
395        // roster to empty and then exits the process, so releasing at the reap
396        // let it exit before the exit handler wrote this child's terminal record
397        // (the stderr drain and snapshot update sit in between), and the
398        // shutdown's own `daemon_shutdown` record was intermittently lost. The
399        // caller releases it after recording the exit (`release_roster`), and
400        // dropping the handle releases it too.
401        let result = self.child.wait().await;
402        #[cfg(target_os = "linux")]
403        if result.is_ok() {
404            if let Some(placement) = self.cgroup_placement.as_ref() {
405                cleanup_reaped_cgroup(placement, &self.module_id).await;
406                // Keep ownership while awaiting kernel population changes: a
407                // drain timeout may cancel this wait and then escalate/reap.
408                self.cgroup_placement = None;
409            }
410        }
411        result
412    }
413
414    #[cfg(target_os = "macos")]
415    async fn confirm_privacy_exec(&mut self) {
416        let Some(pending) = &mut self.privacy_exec else {
417            return;
418        };
419        let result = tokio::time::timeout_at(pending.deadline, async {
420            let mut record = Vec::new();
421            loop {
422                let mut ready = pending.reader.readable().await?;
423                let read = ready.try_io(|reader| {
424                    use std::io::Read;
425                    let mut reader = reader.get_ref();
426                    let mut buffer = [0u8; 256];
427                    reader.read(&mut buffer).map(|count| (count, buffer))
428                });
429                match read {
430                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
431                    Ok(Ok((count, buffer))) => {
432                        if record.len() + count > 1024 {
433                            return Err(io::Error::other(
434                                "privacy exec refusal record is too long",
435                            ));
436                        }
437                        record.extend_from_slice(&buffer[..count]);
438                    }
439                    Ok(Err(error)) => return Err(error),
440                    Err(_) => continue,
441                }
442            }
443        })
444        .await;
445        #[cfg(test)]
446        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
447            use tokio::io::{AsyncReadExt, AsyncWriteExt};
448            if stream.is_none() {
449                *stream = Some(
450                    tokio::net::TcpStream::connect(address.as_str())
451                        .await
452                        .unwrap(),
453                );
454                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
455                stream
456                    .as_mut()
457                    .unwrap()
458                    .write_all(&self.pid.to_ne_bytes())
459                    .await
460                    .unwrap();
461            }
462            let mut reply = [0];
463            stream
464                .as_mut()
465                .unwrap()
466                .read_exact(&mut reply)
467                .await
468                .unwrap();
469            pending.confirmation_pause = None;
470            if &reply == b"F" {
471                Ok(Err(io::Error::other("injected late exec refusal")))
472            } else {
473                assert_eq!(&reply, b"X");
474                result
475            }
476        } else {
477            result
478        };
479        // Keep the reader in self across await: select cancellation must not
480        // discard the handshake or reset its original five-second deadline.
481        let pending = self.privacy_exec.as_ref().expect("pending exec");
482        let cause = match result {
483            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
484            Ok(Err(error)) => Some(format!(
485                "privacy identity exec acknowledgement failed: {error}"
486            )),
487            Ok(Ok(record)) if !record.is_empty() => Some(
488                std::str::from_utf8(&record)
489                    .ok()
490                    .and_then(|record| {
491                        record
492                            .trim()
493                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
494                    })
495                    .filter(|cause| !cause.is_empty())
496                    .unwrap_or("invalid privacy exec refusal record")
497                    .to_string(),
498            ),
499            Ok(Ok(_)) => match self.child.try_wait() {
500                // Empty EOF is the exec acknowledgement. A real module may exit
501                // immediately, including with a reserved trampoline status; no
502                // image is admitted, and its ordinary exit contract stays intact.
503                Ok(Some(_status)) => None,
504                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
505                Ok(None) => {
506                    let image = observe_spawned_image(self.pid);
507                    if let Some(image) = image.filter(|image| {
508                        image.executable.is_some()
509                            && image.executable != pending.trampoline
510                            && (image.executable == pending.expected || pending.script)
511                    }) {
512                        if let Err(error) = self
513                            .roster_guard
514                            .as_ref()
515                            .map(|guard| guard.confirm_executable(image))
516                            .transpose()
517                        {
518                            Some(format!(
519                                "could not durably record confirmed module image: {error}"
520                            ))
521                        } else {
522                            let _ = self.report_ready.set(());
523                            info!(module_id = %pending.module_id, pid = self.pid,
524                                "module spawned with own privacy identity (responsibility disclaimed)");
525                            None
526                        }
527                    } else if image.is_none()
528                        || image.is_some_and(|image| image.executable.is_none())
529                    {
530                        // A process can exit between try_wait and the kernel
531                        // image read. Empty EOF already acknowledged exec, so
532                        // preserve that module's ordinary exit rather than
533                        // mislabel a disappearing image as trampoline refusal.
534                        // Keep pending in self across await so cancellation does
535                        // not discard validation or reset its original deadline.
536                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
537                            Ok(Ok(_status)) => None,
538                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
539                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
540                        }
541                    } else {
542                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
543                    }
544                }
545            },
546        };
547        let pending = self.privacy_exec.take().expect("pending exec");
548        // Exec confirmation describes launch success, not roster retirement.
549        // Keep even a reaped child recorded until its terminal history exists.
550        self.exec.0.finish(if cause.is_some() {
551            crate::activity::ExecState::Refused
552        } else {
553            crate::activity::ExecState::CommittedExec
554        });
555        if let Some(cause) = cause {
556            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
557            self.spawn_failure = Some(cause);
558            // No image is admitted on failure. Reach the entire fresh process
559            // group, including a module which spawned a helper before refusal.
560            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
561                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
562            }
563            let _ = self.child.start_kill();
564        }
565    }
566
567    /// Releases this child's daemon-shutdown roster entry once its exit has
568    /// been recorded. The pid is already reaped and free for reuse, so the
569    /// entry must not outlive the record any longer than that.
570    fn release_roster(&mut self) {
571        self.roster_guard = None;
572    }
573
574    /// Kill the child and, where containment is available, its process tree.
575    ///
576    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
577    /// helper process leaked the helper — the Synapse embedding module's CUDA
578    /// worker holds the GPU allocation, so the leak cost VRAM until the next
579    /// restart of something else. Terminating the job reaches grandchildren that
580    /// a tree walk cannot, including one whose parent has already exited and
581    /// been reparented away.
582    ///
583    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
584    /// direct-child kill still decides the outcome, so containment can never
585    /// change whether a module is reported as stopped.
586    fn start_kill(&mut self) -> io::Result<()> {
587        #[cfg(windows)]
588        if let Some(job) = &self.job {
589            if let Err(error) = job.terminate() {
590                debug!(
591                    error = %error,
592                    "job termination failed; the direct-child kill still owns the outcome"
593                );
594            }
595        }
596        #[cfg(target_os = "linux")]
597        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
598        self.child.start_kill()
599    }
600
601    async fn drain_stderr(&mut self, module_id: &str) {
602        if let Some(pump) = self.stdout_pump.take() {
603            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
604        }
605
606        let Some(pump) = self.stderr_pump.take() else {
607            return;
608        };
609        settle_stderr_pump(
610            module_id,
611            &self.stderr_ring,
612            pump,
613            STDERR_PUMP_DRAIN_TIMEOUT,
614        )
615        .await;
616    }
617}
618
619/// Bound the restart's wait, not the lifetime of the exited child's stdout
620/// reader. Slow pipe reads or capture writes must not turn the bound into lost
621/// output; dropping the handle lets the reader finish at EOF, like stderr.
622async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
623    match timeout(bound, &mut pump).await {
624        Ok(Ok(())) => {}
625        Ok(Err(error)) => {
626            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
627        }
628        Err(_) => {
629            drop(pump);
630            warn!(
631                module_id,
632                waited = ?bound,
633                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
634            );
635        }
636    }
637}
638
639/// The reader task for one process's stderr, with the ring generation its
640/// lines are attributed to.
641struct StderrPump {
642    task: JoinHandle<()>,
643    generation: u64,
644}
645
646/// Retire an exited process's stderr reader and wait up to `bound` for it to
647/// reach EOF. A reader still running at the bound is detached, not stopped: it
648/// keeps filling the exited process's section of the ring until its pipe
649/// closes, and the tail reads `Incomplete` until then. See
650/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
651async fn settle_stderr_pump(
652    module_id: &str,
653    ring: &Arc<Mutex<StderrRing>>,
654    pump: StderrPump,
655    bound: Duration,
656) {
657    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
658    let StderrPump {
659        mut task,
660        generation,
661    } = pump;
662    lock().retire_pump(generation);
663    match timeout(bound, &mut task).await {
664        Ok(Ok(())) => {}
665        Ok(Err(err)) => {
666            let mut ring = lock();
667            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
668            ring.finish_pump(generation);
669            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
670        }
671        Err(_) => {
672            // Dropping the handle detaches the task; it ends at EOF on its pipe.
673            drop(task);
674            lock().mark_pump_late(
675                generation,
676                format!(
677                    "stderr of the exited process had not reached EOF {bound:?} after it was \
678                     retired (a descendant may still hold the pipe open); lines it still \
679                     writes are kept in that process's section"
680                ),
681            );
682            warn!(
683                module_id,
684                waited = ?bound,
685                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
686            );
687        }
688    }
689}
690
691fn registration_release_events() -> &'static watch::Sender<u64> {
692    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
693    EVENTS.get_or_init(|| {
694        let (sender, _receiver) = watch::channel(0);
695        sender
696    })
697}
698
699pub(crate) fn notify_registration_release() {
700    let events = registration_release_events();
701    let next_generation = (*events.borrow()).wrapping_add(1);
702    events.send_replace(next_generation);
703}
704
705/// How to launch one singleton module process.
706#[derive(Debug, Clone, PartialEq, Eq)]
707pub struct ModuleSpec {
708    pub module_id: String,
709    pub program: PathBuf,
710    pub args: Vec<String>,
711    pub env: Vec<(String, String)>,
712    /// When true this is a reserved module: each spawn gets a fresh one-time launch
713    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
714    /// process can register this module_id (a security-boundary module like the
715    /// credential vault must not be impersonable while it is down/restarting).
716    pub reserved: bool,
717    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
718    /// Prefixes come from daemon config and must end in `:` before they reach the
719    /// supervisor; the owner module's current spawn nonce authorizes claims under
720    /// each prefix.
721    pub reserved_prefixes: Vec<String>,
722    /// The wire protocol this module speaks, as DECLARED in daemon config.
723    ///
724    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
725    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
726    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
727    /// and NO launch nonce, and a clean exit the daemon did not request is
728    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
729    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
730    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
731    /// because a process ignores an environment variable it does not read.
732    ///
733    /// The argument is the part that cannot be "harmless to a process that
734    /// ignores it": a stock binary exits on an unknown flag before it listens
735    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
736    /// first conformance run against this mode found it. The nonce is withheld
737    /// because a process that will never present it gains nothing from holding
738    /// it, and a secret in the environment of a process that does not need it is
739    /// a leak surface for no benefit.
740    pub protocol: ModuleProtocol,
741    /// Whether two processes of this module may run at once, which is what a
742    /// blue/green swap does for the length of its overlap. Declared in daemon
743    /// config because the daemon must be able to answer it while the module is
744    /// down, and so a module cannot talk itself into it after registering.
745    pub overlap: ModuleOverlap,
746}
747
748/// Only the configured launch inputs enter this value; spawned connection paths,
749/// nonces and inherited environment never do. Its bytes remain daemon-private.
750#[derive(Clone, PartialEq, Eq)]
751pub(crate) struct LaunchSpecDigest([u8; 32]);
752
753impl fmt::Debug for LaunchSpecDigest {
754    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
755        f.write_str("LaunchSpecDigest([redacted])")
756    }
757}
758
759impl LaunchSpecDigest {
760    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
761        fn field(hash: &mut Sha256, bytes: &[u8]) {
762            hash.update((bytes.len() as u64).to_le_bytes());
763            hash.update(bytes);
764        }
765        let mut hash = Sha256::new();
766        if let Some(program) = spec.program.to_str() {
767            field(&mut hash, b"utf8-program");
768            field(&mut hash, program.as_bytes());
769        } else {
770            // The daemon config is UTF-8, but in-process callers may provide
771            // native paths: lossy conversion would collapse distinct programs.
772            #[cfg(unix)]
773            {
774                use std::os::unix::ffi::OsStrExt;
775                field(&mut hash, b"unix-program");
776                field(&mut hash, spec.program.as_os_str().as_bytes());
777            }
778            #[cfg(windows)]
779            {
780                use std::os::windows::ffi::OsStrExt;
781                field(&mut hash, b"windows-program");
782                let bytes = spec
783                    .program
784                    .as_os_str()
785                    .encode_wide()
786                    .flat_map(u16::to_le_bytes)
787                    .collect::<Vec<_>>();
788                field(&mut hash, &bytes);
789            }
790            #[cfg(not(any(unix, windows)))]
791            field(&mut hash, spec.program.to_string_lossy().as_bytes());
792        }
793        hash.update((spec.args.len() as u64).to_le_bytes());
794        for arg in &spec.args {
795            field(&mut hash, arg.as_bytes());
796        }
797        let mut env = spec.env.iter().collect::<Vec<_>>();
798        env.sort_unstable();
799        hash.update((env.len() as u64).to_le_bytes());
800        for (key, value) in env {
801            field(&mut hash, key.as_bytes());
802            field(&mut hash, value.as_bytes());
803        }
804        field(
805            &mut hash,
806            match spec.protocol {
807                ModuleProtocol::Subc => b"subc",
808                ModuleProtocol::None => b"none",
809            },
810        );
811        Self(hash.finalize().into())
812    }
813}
814
815/// Whether a module tolerates a second process of itself running alongside.
816///
817/// Most modules are single-writer on their store (a WAL, a capture log, a
818/// resident index behind a writer barrier), and two processes on one store
819/// corrupt it. So a swap, which overlaps the old and new process by design,
820/// is refused unless the module's config opts in with `overlap: "safe"`.
821#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
822pub enum ModuleOverlap {
823    /// Never run two processes of this module at once. The default.
824    #[default]
825    Exclusive,
826    /// The module has said a second process of itself is harmless for the
827    /// length of a swap.
828    ///
829    /// Declare it only if a second instance can run for a few seconds without
830    /// touching ANY single-writer store: every database, WAL, index, projector
831    /// and scheduled job the module owns. A lease on part of that state is not
832    /// enough. broca's session lease guards WAL appends while its run index, its
833    /// store projector and its archive fold timer (which unlinks live WAL files)
834    /// stay single-writer, so broca is exclusive despite holding a lease. The
835    /// refusal only fires after this has been decided, so the decision is the
836    /// check.
837    Safe,
838}
839
840impl ModuleOverlap {
841    pub fn as_str(self) -> &'static str {
842        match self {
843            Self::Exclusive => "exclusive",
844            Self::Safe => "safe",
845        }
846    }
847}
848
849/// Environment variable telling a spawned module which case it was started
850/// for, before it sends HELLO. Only a swap candidate carries it, as
851/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
852///
853/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
854/// longer because nobody waits on it, while a plain restart must flip ready
855/// quickly because callers see `module_warming` until it does. Absence means
856/// plain restart, the safe reading. The daemon trusts nothing about it; the
857/// candidate is proven by its launch nonce at HELLO.
858pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
859/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
860pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
861/// How long a swap waits for its candidate to register and declare itself
862/// ready when the operator does not say. A module warming as a swap candidate
863/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
864/// daemon allows that plus time to start the process and send HELLO.
865pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
866
867/// Bounded restart policy for crash exits.
868///
869/// `max_restarts` is the number of replacement processes allowed after the
870/// initial spawn WITHIN `window`. After that many crash restarts inside one
871/// window the module enters [`ModuleState::Failed`] and the supervisor stops
872/// the crash loop.
873///
874/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
875/// and that only survived because crashes were rare: a module that crashed
876/// three times across a week was disabled forever by crashes that had nothing
877/// to do with each other. That stopped being survivable once modules began
878/// exiting non-zero whenever the daemon's connection to them drops, because
879/// then every daemon-side connection drop spends a unit of the same budget and
880/// one flappy hour permanently stops a healthy module. Restarts older than
881/// `window` release their slot, so a module that crashed twice yesterday has a
882/// full budget today, while a genuine crash loop -- which is fast by
883/// definition -- still reaches the cap and stops.
884#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct RestartPolicy {
886    pub max_restarts: u32,
887    /// Base delay before a crash replacement. The actual delay escalates with
888    /// the number of recent crash replacements and is capped by `max_backoff`.
889    pub backoff: Duration,
890    /// Maximum delay before a crash replacement.
891    pub max_backoff: Duration,
892    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
893    /// budget effectively infinite (nothing is ever in-window), which is why
894    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
895    pub window: Duration,
896}
897
898impl RestartPolicy {
899    /// A policy with the default crash window. Callers that care about the
900    /// window say so with [`Self::with_window`]; the ones that do not are
901    /// asking for the standard rate limit, not for no limit.
902    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
903        Self {
904            max_restarts,
905            backoff,
906            max_backoff: DEFAULT_MAX_BACKOFF,
907            window: DEFAULT_RESTART_WINDOW,
908        }
909    }
910
911    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
912        self.max_backoff = max_backoff;
913        self
914    }
915
916    pub fn with_window(mut self, window: Duration) -> Self {
917        self.window = window;
918        self
919    }
920
921    /// Calculate the capped exponential delay for the next crash replacement.
922    /// `restart_in_window` is zero for the first replacement after an operator
923    /// action (restart, reload, re-enable) cleared the crash ring, or after all
924    /// older crash replacements have aged out of the window.
925    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
926        if self.backoff.is_zero() || self.max_backoff.is_zero() {
927            return Duration::ZERO;
928        }
929
930        let mut delay = self.backoff;
931        for _ in 0..restart_in_window {
932            if delay >= self.max_backoff {
933                return self.max_backoff;
934            }
935            delay = delay
936                .checked_mul(10)
937                .unwrap_or(self.max_backoff)
938                .min(self.max_backoff);
939        }
940        delay.min(self.max_backoff)
941    }
942
943    /// The one sentence that explains a budget-exhausted stop, used for both the
944    /// log line and the terminal record so the two cannot drift. It names the
945    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
946    /// exactly what this budget is not.
947    fn budget_exhausted_detail(&self) -> String {
948        format!(
949            "crash budget exhausted: max_restarts={} within window_secs={}",
950            self.max_restarts,
951            self.window.as_secs()
952        )
953    }
954}
955
956impl Default for RestartPolicy {
957    fn default() -> Self {
958        Self {
959            max_restarts: DEFAULT_MAX_RESTARTS,
960            backoff: DEFAULT_BACKOFF,
961            max_backoff: DEFAULT_MAX_BACKOFF,
962            window: DEFAULT_RESTART_WINDOW,
963        }
964    }
965}
966
967#[derive(Debug, Clone, Copy, PartialEq, Eq)]
968struct CrashRestartSchedule {
969    restart_in_window: u32,
970    delay: Duration,
971}
972
973/// Whether the daemon itself will bring this module back after the exit being
974/// handled: it is enabled AND its in-window crash restarts are below the cap.
975///
976/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
977/// the window are dropped here rather than by a timer, so the count is right
978/// the moment somebody asks and no bookkeeping runs for idle modules.
979fn daemon_will_restart(
980    state: &mut SupervisorSnapshot,
981    policy: &RestartPolicy,
982    now: Instant,
983) -> bool {
984    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
985}
986
987const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
988const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
989const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
990const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
991
992#[derive(Debug, Clone, Copy, PartialEq, Eq)]
993pub enum HealthAction {
994    Report,
995    Restart,
996    Alert,
997}
998
999impl fmt::Display for HealthAction {
1000    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1001        f.write_str(match self {
1002            Self::Report => "report",
1003            Self::Restart => "restart",
1004            Self::Alert => "alert",
1005        })
1006    }
1007}
1008
1009#[derive(Debug, Clone, PartialEq, Eq)]
1010pub struct HealthConfig {
1011    /// Optional loopback HTTP endpoint for a managed non-wire process.
1012    /// Changing it applies live on rescan; the process protocol changes only
1013    /// at its next spawn.
1014    pub http: Option<String>,
1015    pub cadence: Duration,
1016    pub deadline: Duration,
1017    pub failure_threshold: u32,
1018    pub on_degraded: HealthAction,
1019    pub on_failing: HealthAction,
1020    pub critical: bool,
1021}
1022
1023impl Default for HealthConfig {
1024    fn default() -> Self {
1025        Self {
1026            http: None,
1027            cadence: DEFAULT_HEALTH_CADENCE,
1028            deadline: DEFAULT_HEALTH_DEADLINE,
1029            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1030            on_degraded: HealthAction::Report,
1031            on_failing: HealthAction::Report,
1032            critical: false,
1033        }
1034    }
1035}
1036
1037/// The supervisor's view of one module's health, relayed to clients over
1038/// channel-0 and rendered by `ck health`.
1039///
1040/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1041/// stated here rather than only at the wire type a consumer reads. A reader can
1042/// look up what `None` means; only a writer can silently change it, and the
1043/// writer has no reason to go looking at a downstream contract before editing.
1044///
1045/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1046/// back to `None` on re-registration precisely so a respawned module does not
1047/// carry its predecessor's timestamp — so an old value and an absent one call for
1048/// opposite readings, and anything that defaulted this to a number would make a
1049/// never-probed module indistinguishable from one probed at the epoch.
1050///
1051/// `detail` and `metrics` are `None` when the module published none on this
1052/// probe, which does not mean it reported nothing wrong — it is also the shape
1053/// when the probe never reached it. `last_probe_ms` is what separates those.
1054#[derive(Debug, Clone, PartialEq)]
1055pub struct ModuleHealthStatus {
1056    pub status: SupervisorHealthStatus,
1057    pub last_probe_ms: Option<u64>,
1058    pub detail: Option<String>,
1059    pub metrics: Option<Value>,
1060    pub consecutive_failures: u32,
1061    /// Number of replies received after a recurring health probe's deadline.
1062    /// Unlike a timeout, every increment proves the module was alive.
1063    pub late_answer_count: u64,
1064    /// End-to-end latency of the newest late reply, measured from probe start.
1065    pub last_late_answer_latency_ms: Option<u64>,
1066    pub last_action: Option<String>,
1067    /// Set together with `last_action`; the pair moves as one, and both being
1068    /// absent means no escalation has ever been taken rather than that the last
1069    /// one succeeded.
1070    pub last_action_ms: Option<u64>,
1071}
1072
1073impl Default for ModuleHealthStatus {
1074    fn default() -> Self {
1075        Self {
1076            status: SupervisorHealthStatus::Unknown,
1077            last_probe_ms: None,
1078            detail: None,
1079            metrics: None,
1080            consecutive_failures: 0,
1081            late_answer_count: 0,
1082            last_late_answer_latency_ms: None,
1083            last_action: None,
1084            last_action_ms: None,
1085        }
1086    }
1087}
1088
1089/// Typed lifecycle state for a supervised module.
1090#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1091pub enum ModuleState {
1092    Starting,
1093    Running,
1094    Unresponsive,
1095    Restarting,
1096    Draining,
1097    Stopped,
1098    Failed,
1099    Disabled,
1100}
1101
1102impl fmt::Display for ModuleState {
1103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1104        f.write_str(match self {
1105            Self::Starting => "starting",
1106            Self::Running => "running",
1107            Self::Unresponsive => "unresponsive",
1108            Self::Restarting => "restarting",
1109            Self::Draining => "draining",
1110            Self::Stopped => "stopped",
1111            Self::Failed => "failed",
1112            Self::Disabled => "disabled",
1113        })
1114    }
1115}
1116
1117/// Supervisor classification of a child-process exit.
1118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1119pub enum ExitKind {
1120    Clean,
1121    Crash,
1122    DeliberateSeverance,
1123}
1124
1125impl From<ExitKind> for TerminalExitKind {
1126    fn from(kind: ExitKind) -> Self {
1127        match kind {
1128            ExitKind::Clean => Self::Clean,
1129            ExitKind::Crash => Self::Crash,
1130            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1131        }
1132    }
1133}
1134
1135/// Exact process identity retained when a supervised module registers its
1136/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1137#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1138pub(crate) struct ProcessIdentity {
1139    pub(crate) pid: u32,
1140    pub(crate) start_time: u64,
1141}
1142
1143/// Last observed child exit, if any.
1144#[derive(Debug, Clone, PartialEq, Eq)]
1145pub struct ExitReport {
1146    pub kind: ExitKind,
1147    pub code: Option<i32>,
1148    pub signal: Option<i32>,
1149    pub at_ms: u64,
1150}
1151
1152/// Point-in-time module status answerable by subc without forwarding to the
1153/// module process.
1154#[derive(Debug, Clone, PartialEq)]
1155pub struct ModuleStatus {
1156    pub module_id: String,
1157    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1158    pub waiting_for_previous_pid: Option<u32>,
1159    /// Why the previous instance still prevents a new start.
1160    pub waiting_for_previous_detail: Option<String>,
1161    /// The daemon's Windows per-spawn record: `pipe` after the process it
1162    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1163    /// List/status reports use this record, not the source the module sends
1164    /// in its registration message's manifest.provenance.launch_nonce_source.
1165    pub launch_nonce_source: Option<String>,
1166    pub state: ModuleState,
1167    pub enabled: bool,
1168    pub process_alive: bool,
1169    pub registration_active: bool,
1170    /// The module's declared wire protocol, carried beside `live` because it is
1171    /// what makes `live` readable: the two fields answer one question together.
1172    /// While a process is alive this is its launch declaration, not a later
1173    /// pending-reload edit. When down it is the configured next launch protocol.
1174    pub protocol: ModuleProtocol,
1175    /// Whether the module is serving, under the strongest definition the daemon
1176    /// can assert for its protocol.
1177    ///
1178    /// A subc module must also be REGISTERED: its process being alive says
1179    /// nothing about whether it can take a request. A `protocol: "none"` module
1180    /// never registers, so that term is dropped and this falls back to "enabled,
1181    /// running, and the process the daemon launched is alive" -- which is all
1182    /// the daemon observes about a process that speaks no subc wire. It stays a
1183    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1184    /// rather than printing it bare.
1185    pub live: bool,
1186    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1187    /// restarts have already released their slot, so this count can go down
1188    /// without anybody touching the module.
1189    pub restart_count: u32,
1190    /// Replacement processes spawned over this module's entire supervisor lifetime;
1191    /// unlike `restart_count`, this value is never reset by an operator action
1192    /// and never falls out of a window.
1193    pub lifetime_restarts: u32,
1194    pub spawn_generation: u64,
1195    /// The budget `restart_count` is spent against. Carried alongside the count
1196    /// because the count alone does not say how close the module is to being
1197    /// disabled, and reporting one without the other is what makes an
1198    /// about-to-be-retired module look ordinary.
1199    pub max_restarts: u32,
1200    /// The span `restart_count` is counted over. Carried with the pair above for
1201    /// the same reason they are carried together: "2 of 3" means one thing for a
1202    /// ten-minute window and something else entirely for a lifetime.
1203    pub restart_window: Duration,
1204    /// Effective drain and restart timing policy used by this running module.
1205    /// These values are carried together with the restart budget so status
1206    /// readers can compare configured intent with what the supervisor applied.
1207    pub drain_timeout: Duration,
1208    pub restart_backoff: Duration,
1209    pub restart_max_backoff: Duration,
1210    /// The module's process. On macOS this stays absent while the `ck-subc`
1211    /// launch trampoline is still running in that pid, and appears once the
1212    /// exec acknowledgement confirms the module image has replaced it. Launch
1213    /// time and the supervisor's own hold on the process are unaffected.
1214    pub pid: Option<u32>,
1215    pub spawned_at_ms: Option<u64>,
1216    pub spawned_from: Option<PathBuf>,
1217    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1218    pub process_start_time: Option<u64>,
1219    pub last_exit: Option<ExitReport>,
1220    pub health: ModuleHealthStatus,
1221}
1222
1223#[cfg(test)]
1224#[derive(Debug, Clone, Copy, PartialEq)]
1225struct ActorSelectCheckpoint {
1226    generation: u64,
1227    turn: u64,
1228    registered_connection: Option<ConnectionId>,
1229    next_probe_at: Option<Instant>,
1230    wake_after: Option<Duration>,
1231}
1232
1233#[derive(Debug, Clone, PartialEq)]
1234struct SupervisorSnapshot {
1235    failed_retry: FailedRetry,
1236    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1237    #[cfg(windows)]
1238    pipe_consumed: Option<Arc<OnceLock<()>>>,
1239    /// Counts running-child loop turns, not executor polls of a parked wait.
1240    #[cfg(test)]
1241    actor_turns: u64,
1242    /// Running select after exec confirmation and with no unconsumed registry
1243    /// event. Recording a spawn or starting a loop turn is not this barrier.
1244    #[cfg(test)]
1245    actor_select: Option<ActorSelectCheckpoint>,
1246    state: ModuleState,
1247    enabled: bool,
1248    process_alive: bool,
1249    spawned_protocol: Option<ModuleProtocol>,
1250    spawn_failure: Option<String>,
1251    waiting_for_previous_pid: Option<u32>,
1252    waiting_for_previous_detail: Option<String>,
1253    /// When each crash restart was spent, oldest first. This IS the crash
1254    /// budget: its in-window length is the count an operator sees and the count
1255    /// the restart decision is made against, so there is no second counter that
1256    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1257    /// operator actions that used to zero the old lifetime counter.
1258    crash_restarts: VecDeque<Instant>,
1259    lifetime_restarts: u32,
1260    /// Successful child spawns in this daemon incarnation.
1261    ///
1262    /// `lifetime_restarts` was considered and rejected: it starts at zero
1263    /// (line 640), successful initial/operator spawns in `set_running` do not
1264    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1265    /// increments before a successful replacement exists (lines 604, 3846,
1266    /// and 3921), so a failed spawn can consume it. This counter moves only
1267    /// when a live PID is accepted below.
1268    spawn_generation: u64,
1269    pid: Option<u32>,
1270    #[cfg(target_os = "macos")]
1271    report_ready: Option<Arc<OnceLock<()>>>,
1272    /// Last reaped child, retained after current process facts are cleared.
1273    reaped_pid: Option<u32>,
1274    /// Whether a replacement has already been scheduled for the current process.
1275    respawn_pending: bool,
1276    /// A second restart is waiting for the replacement already scheduled.
1277    coalesced_restart_pending: bool,
1278    spawned_at_ms: Option<u64>,
1279    spawned_from: Option<PathBuf>,
1280    launch_spec_digest: Option<LaunchSpecDigest>,
1281    spawned_file_identity: Option<SpawnedFileIdentity>,
1282    process_start_time: Option<u64>,
1283    deliberate_severance: Option<ProcessIdentity>,
1284    last_exit: Option<ExitReport>,
1285    /// Diagnostic attached to the next drain's terminal record, if any.
1286    drain_disposition_detail: Option<String>,
1287    health: ModuleHealthStatus,
1288    /// Whether the current process was started as a swap candidate and so
1289    /// lives in the module's alternate cgroup. The next swap's candidate takes
1290    /// the other one, so the two processes of a swap never share a cgroup. A
1291    /// plain spawn always uses the primary cgroup.
1292    in_alternate_slot: bool,
1293    /// Whether the current `Draining` state ends in a replacement process
1294    /// (restart, reload, health restart) rather than a stop. Only meaningful
1295    /// while `state` is `Draining`; every entry into that state rewrites it.
1296    /// It is what lets route.open answer the retryable `module_reloading` to a
1297    /// consumer that reaches a still-registered process mid-restart, instead of
1298    /// the `supervisor_not_live` a stop or disable deserves.
1299    draining_to_replace: bool,
1300    /// Whether a configuration update has been applied since the current
1301    /// process was spawned, so that process runs an older spec than the one
1302    /// the supervisor now holds. A queued restart is only coalesced into a
1303    /// fresher process when this is false: a restart requested to pick up a
1304    /// new configuration must not be satisfied by a process that predates it.
1305    configuration_updated_since_spawn: bool,
1306}
1307
1308impl SupervisorSnapshot {
1309    /// The pid that status, provenance and resource readings may report. While
1310    /// the launch trampoline still runs in the pid, reading its executable or
1311    /// resource use would describe `ck-subc`, not the module, so none is
1312    /// reported until the exec acknowledgement confirms the module image.
1313    fn reported_pid(&self) -> Option<u32> {
1314        #[cfg(target_os = "macos")]
1315        if self
1316            .report_ready
1317            .as_ref()
1318            .is_some_and(|ready| ready.get().is_none())
1319        {
1320            return None;
1321        }
1322        self.pid
1323    }
1324
1325    fn starting() -> Self {
1326        Self::new(ModuleState::Starting, true)
1327    }
1328
1329    fn disabled() -> Self {
1330        Self::new(ModuleState::Disabled, false)
1331    }
1332
1333    /// Crash restarts still inside `window`, having dropped the ones that are
1334    /// not. Pruning on read is what makes the budget a rate: an instant older
1335    /// than the window stops holding a slot the moment anybody counts.
1336    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1337        while let Some(oldest) = self.crash_restarts.front() {
1338            if now.duration_since(*oldest) > window {
1339                self.crash_restarts.pop_front();
1340            } else {
1341                break;
1342            }
1343        }
1344        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1345    }
1346
1347    /// Spend one unit of the crash budget and record the restart in the ledger.
1348    ///
1349    /// The ring is bounded by the cap because more than `max_restarts` in-window
1350    /// instants can never be reached (the caller refuses the restart first), so
1351    /// anything beyond that is an unbounded queue waiting to happen.
1352    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1353        self.crash_restarts.push_back(now);
1354        while self.crash_restarts.len() > policy.max_restarts as usize {
1355            self.crash_restarts.pop_front();
1356        }
1357        self.lifetime_restarts += 1;
1358    }
1359
1360    /// Reserve one crash-restart slot and calculate the delay before respawning.
1361    /// The count is captured before recording this restart, so the first retry
1362    /// uses the base delay and each later in-window retry escalates once.
1363    fn next_crash_restart(
1364        &mut self,
1365        policy: &RestartPolicy,
1366        now: Instant,
1367    ) -> Option<CrashRestartSchedule> {
1368        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1369        if restart_in_window >= policy.max_restarts {
1370            return None;
1371        }
1372        self.record_crash_restart(policy, now);
1373        Some(CrashRestartSchedule {
1374            restart_in_window,
1375            delay: policy.delay_for_restart(restart_in_window),
1376        })
1377    }
1378
1379    /// Give the module its full budget back, as an operator restart, reload, or
1380    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1381    /// ledger of what actually happened, and an operator action does not unmake
1382    /// the crashes.
1383    fn clear_crash_restarts(&mut self) {
1384        self.crash_restarts.clear();
1385    }
1386
1387    fn new(state: ModuleState, enabled: bool) -> Self {
1388        Self {
1389            failed_retry: FailedRetry::default(),
1390            exec_outcome: None,
1391            #[cfg(test)]
1392            actor_turns: 0,
1393            #[cfg(test)]
1394            actor_select: None,
1395            state,
1396            enabled,
1397            process_alive: false,
1398            #[cfg(windows)]
1399            pipe_consumed: None,
1400            spawned_protocol: None,
1401            spawn_failure: None,
1402            waiting_for_previous_pid: None,
1403            waiting_for_previous_detail: None,
1404            crash_restarts: VecDeque::new(),
1405            lifetime_restarts: 0,
1406            spawn_generation: 0,
1407            pid: None,
1408            #[cfg(target_os = "macos")]
1409            report_ready: None,
1410            reaped_pid: None,
1411            respawn_pending: false,
1412            coalesced_restart_pending: false,
1413            spawned_at_ms: None,
1414            spawned_from: None,
1415            launch_spec_digest: None,
1416            spawned_file_identity: None,
1417            process_start_time: None,
1418            deliberate_severance: None,
1419            last_exit: None,
1420            drain_disposition_detail: None,
1421            health: ModuleHealthStatus::default(),
1422            in_alternate_slot: false,
1423            draining_to_replace: false,
1424            configuration_updated_since_spawn: false,
1425        }
1426    }
1427}
1428
1429type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1430
1431type SpawnSubscriberKey = (ConnectionId, u64);
1432
1433#[derive(Debug)]
1434struct SpawnSubscriber {
1435    version: u8,
1436    frames: mpsc::Sender<Frame>,
1437    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1438    /// from which event. The full frame channel cannot carry that news, so it
1439    /// travels beside it; see `SpawnEventFeed::subscribe`.
1440    lagged: Option<oneshot::Sender<SpawnCursor>>,
1441}
1442
1443#[derive(Debug)]
1444struct SpawnEventState {
1445    daemon_incarnation: String,
1446    seq: u64,
1447    capacity: usize,
1448    live: HashMap<String, LiveSpawn>,
1449    generations: HashMap<String, u64>,
1450    events: VecDeque<SpawnEvent>,
1451    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1452}
1453
1454impl Default for SpawnEventState {
1455    fn default() -> Self {
1456        Self {
1457            daemon_incarnation: "unconfigured".to_string(),
1458            seq: 0,
1459            capacity: SPAWN_EVENT_RING_CAPACITY,
1460            live: HashMap::new(),
1461            generations: HashMap::new(),
1462            events: VecDeque::new(),
1463            subscribers: HashMap::new(),
1464        }
1465    }
1466}
1467
1468#[derive(Debug, Clone, Default)]
1469struct SpawnEventFeed(
1470    Arc<Mutex<SpawnEventState>>,
1471    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1472);
1473
1474#[derive(Debug, Clone, PartialEq, Eq)]
1475pub(crate) enum SpawnSubscribeRefusal {
1476    ForeignIncarnation { current: String },
1477    TooOld { oldest: SpawnCursor },
1478    Frame(String),
1479}
1480
1481impl SpawnEventFeed {
1482    fn configure_incarnation(&self, daemon_incarnation: String) {
1483        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1484        state.daemon_incarnation = daemon_incarnation;
1485        state.seq = 0;
1486        state.live.clear();
1487        state.generations.clear();
1488        state.events.clear();
1489        state.subscribers.clear();
1490    }
1491
1492    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1493        SpawnCursor {
1494            daemon_incarnation: state.daemon_incarnation.clone(),
1495            seq: state.seq,
1496        }
1497    }
1498
1499    fn snapshot(&self) -> SpawnSnapshot {
1500        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1501        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1502        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1503        SpawnSnapshot {
1504            cursor: Self::cursor(&state),
1505            ring_bound: state.capacity as u64,
1506            live,
1507        }
1508    }
1509
1510    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1511        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1512        let generation = state
1513            .generations
1514            .get(module_id)
1515            .copied()
1516            .unwrap_or(0)
1517            .checked_add(1)
1518            .expect("spawn generation exhausted");
1519        state.generations.insert(module_id.to_string(), generation);
1520        let live = LiveSpawn {
1521            module_id: module_id.to_string(),
1522            spawn_generation: generation,
1523            pid,
1524            spawned_at_ms,
1525        };
1526        state.live.insert(module_id.to_string(), live);
1527        Self::emit_locked(
1528            &mut state,
1529            SpawnEventKind::Spawned,
1530            module_id.to_string(),
1531            generation,
1532            pid,
1533            None,
1534            None,
1535        );
1536        generation
1537    }
1538
1539    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1540        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1541        let Some(live) = state.live.remove(module_id) else {
1542            warn!(
1543                module_id,
1544                "terminal record had no live spawn event identity"
1545            );
1546            return;
1547        };
1548        Self::emit_locked(
1549            &mut state,
1550            SpawnEventKind::Exited,
1551            module_id.to_string(),
1552            live.spawn_generation,
1553            live.pid,
1554            exit_code,
1555            exit_signal,
1556        );
1557    }
1558
1559    /// Report the exit of a process that a swap has already replaced.
1560    ///
1561    /// `emit_exited` removes the module's live entry, which after a swap's
1562    /// cutover describes the promoted candidate, not the old process now
1563    /// exiting. This emits the old generation's exit and leaves the live entry
1564    /// alone unless it still names that generation.
1565    fn emit_superseded_exited(
1566        &self,
1567        module_id: &str,
1568        spawn_generation: u64,
1569        pid: u32,
1570        exit_code: Option<i32>,
1571        exit_signal: Option<i32>,
1572    ) {
1573        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1574        if state
1575            .live
1576            .get(module_id)
1577            .is_some_and(|live| live.spawn_generation == spawn_generation)
1578        {
1579            state.live.remove(module_id);
1580        }
1581        Self::emit_locked(
1582            &mut state,
1583            SpawnEventKind::Exited,
1584            module_id.to_string(),
1585            spawn_generation,
1586            pid,
1587            exit_code,
1588            exit_signal,
1589        );
1590    }
1591
1592    #[allow(clippy::too_many_arguments)]
1593    fn emit_locked(
1594        state: &mut SpawnEventState,
1595        kind: SpawnEventKind,
1596        module_id: String,
1597        spawn_generation: u64,
1598        pid: u32,
1599        exit_code: Option<i32>,
1600        exit_signal: Option<i32>,
1601    ) {
1602        state.seq = state
1603            .seq
1604            .checked_add(1)
1605            .expect("spawn event sequence exhausted");
1606        let event = SpawnEvent {
1607            cursor: Self::cursor(state),
1608            kind,
1609            module_id,
1610            spawn_generation,
1611            pid,
1612            exit_code,
1613            exit_signal,
1614        };
1615        state.events.push_back(event.clone());
1616        while state.events.len() > state.capacity {
1617            state.events.pop_front();
1618        }
1619        let body = match serde_json::to_vec(&event) {
1620            Ok(body) => body,
1621            Err(error) => {
1622                error!(%error, "failed to serialize supervisor spawn event");
1623                return;
1624            }
1625        };
1626        state.subscribers.retain(|(connection_id, corr), subscriber| {
1627            let frame = Frame::build_with_version(
1628                subscriber.version,
1629                FrameType::StreamData,
1630                control_flags(),
1631                0,
1632                0,
1633                *corr,
1634                body.clone(),
1635            );
1636            match frame {
1637                Ok(frame) => {
1638                    if subscriber.frames.try_send(frame).is_ok() {
1639                        true
1640                    } else {
1641                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1642                        if let Some(lagged) = subscriber.lagged.take() {
1643                            let _ = lagged.send(event.cursor.clone());
1644                        }
1645                        false
1646                    }
1647                }
1648                Err(error) => {
1649                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1650                    false
1651                }
1652            }
1653        });
1654    }
1655
1656    fn subscribe(
1657        &self,
1658        connection_id: ConnectionId,
1659        corr: u64,
1660        version: u8,
1661        since: Option<SpawnCursor>,
1662        sink: FrameSink,
1663    ) -> Result<(), SpawnSubscribeRefusal> {
1664        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1665        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1666        {
1667            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1668            let replay = if let Some(since) = since {
1669                if since.daemon_incarnation != state.daemon_incarnation {
1670                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1671                        current: state.daemon_incarnation.clone(),
1672                    });
1673                }
1674                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1675                    if since.seq < oldest.seq.saturating_sub(1) {
1676                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1677                    }
1678                }
1679                state
1680                    .events
1681                    .iter()
1682                    .filter(|event| event.cursor.seq > since.seq)
1683                    .cloned()
1684                    .collect::<Vec<_>>()
1685            } else {
1686                Vec::new()
1687            };
1688            for event in replay {
1689                let body = serde_json::to_vec(&event)
1690                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1691                let frame = Frame::build_with_version(
1692                    version,
1693                    FrameType::StreamData,
1694                    control_flags(),
1695                    0,
1696                    0,
1697                    corr,
1698                    body,
1699                )
1700                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1701                frames
1702                    .try_send(frame)
1703                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1704            }
1705            state.subscribers.insert(
1706                (connection_id, corr),
1707                SpawnSubscriber {
1708                    version,
1709                    frames,
1710                    lagged: Some(lagged),
1711                },
1712            );
1713        }
1714        // The lagged terminal is sent here, by the forwarder, rather than by
1715        // the emitter: at the moment of the drop the subscriber's own channel
1716        // is full, and writing to the connection sink directly from the emitter
1717        // would put the Error AHEAD of the events still queued in that channel
1718        // (and the emitter holds the feed lock, so it cannot await the sink).
1719        // Dropping the subscriber drops the only sender, so `recv` drains every
1720        // queued event and then returns `None`; only then is the Error sent, so
1721        // the client sees each event it can keep, then the reason it was cut.
1722        // Cancel and connection removal drop the oneshot unsent, so they end
1723        // the stream with no Error.
1724        tokio::spawn(async move {
1725            while let Some(frame) = receiver.recv().await {
1726                if sink.send(frame).await.is_err() {
1727                    return;
1728                }
1729            }
1730            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1731                return;
1732            };
1733            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1734                Ok(frame) => {
1735                    let _ = sink.send(frame).await;
1736                }
1737                Err(error) => {
1738                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1739                }
1740            }
1741        });
1742        Ok(())
1743    }
1744
1745    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1746        let Some(subscriber) = self
1747            .0
1748            .lock()
1749            .unwrap_or_else(|p| p.into_inner())
1750            .subscribers
1751            .remove(&(connection_id, corr))
1752        else {
1753            return false;
1754        };
1755        if let Ok(frame) = Frame::build_with_version(
1756            subscriber.version,
1757            FrameType::StreamEnd,
1758            control_flags(),
1759            0,
1760            0,
1761            corr,
1762            Vec::new(),
1763        ) {
1764            tokio::spawn(async move {
1765                let _ = subscriber.frames.send(frame).await;
1766            });
1767        }
1768        true
1769    }
1770
1771    fn remove_connection(&self, connection_id: ConnectionId) {
1772        self.0
1773            .lock()
1774            .unwrap_or_else(|p| p.into_inner())
1775            .subscribers
1776            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1777    }
1778
1779    #[cfg(any(test, feature = "test-support"))]
1780    fn set_capacity(&self, capacity: usize) {
1781        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1782    }
1783
1784    #[cfg(any(test, feature = "test-support"))]
1785    fn subscriber_count(&self) -> usize {
1786        self.0
1787            .lock()
1788            .unwrap_or_else(|p| p.into_inner())
1789            .subscribers
1790            .len()
1791    }
1792}
1793
1794/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1795/// The terminal Error a lagged spawn subscriber receives after its queued events.
1796fn spawn_subscriber_lagged_frame(
1797    version: u8,
1798    corr: u64,
1799    first_undelivered: SpawnCursor,
1800) -> Result<Frame, String> {
1801    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1802        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1803        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1804            .to_string(),
1805        detail: Some(serde_json::json!({
1806            "first_undelivered_cursor": first_undelivered
1807        })),
1808    })
1809    .map_err(|error| error.to_string())?;
1810    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1811        .map_err(|error| error.to_string())
1812}
1813
1814pub trait ModuleProcessLiveness: Send + Sync {
1815    fn process_live(&self, module_id: &str) -> Option<bool>;
1816
1817    /// Whether the supervisor is replacing this module's process right now: an
1818    /// operator restart or reload, a health restart, or a crash respawn whose
1819    /// backoff is running. A module in that state is not live, but a consumer
1820    /// refused now should retry shortly rather than treat the target as gone.
1821    /// Stopped, failed, and disabled modules are not replacing.
1822    fn process_replacing(&self, _module_id: &str) -> bool {
1823        false
1824    }
1825}
1826
1827/// Shared process-liveness registry keyed by supervised `module_id`.
1828#[derive(Debug, Clone, Default)]
1829pub struct SupervisorProcessLiveness {
1830    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1831}
1832
1833impl SupervisorProcessLiveness {
1834    pub fn new() -> Self {
1835        Self::default()
1836    }
1837
1838    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        snapshots.insert(module_id, snapshot);
1844    }
1845
1846    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1847        let mut snapshots = self
1848            .snapshots
1849            .lock()
1850            .unwrap_or_else(|poisoned| poisoned.into_inner());
1851        let is_current = snapshots
1852            .get(module_id)
1853            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1854            .unwrap_or(false);
1855        if is_current {
1856            snapshots.remove(module_id);
1857        }
1858    }
1859}
1860
1861impl ModuleProcessLiveness for SupervisorProcessLiveness {
1862    fn process_live(&self, module_id: &str) -> Option<bool> {
1863        let snapshot = {
1864            let snapshots = self
1865                .snapshots
1866                .lock()
1867                .unwrap_or_else(|poisoned| poisoned.into_inner());
1868            snapshots.get(module_id).cloned()
1869        }?;
1870        let snapshot = snapshot
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner());
1873        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1874    }
1875
1876    fn process_replacing(&self, module_id: &str) -> bool {
1877        let Some(snapshot) = self
1878            .snapshots
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner())
1881            .get(module_id)
1882            .cloned()
1883        else {
1884            return false;
1885        };
1886        let snapshot = snapshot
1887            .lock()
1888            .unwrap_or_else(|poisoned| poisoned.into_inner());
1889        snapshot.enabled
1890            && match snapshot.state {
1891                ModuleState::Restarting => true,
1892                ModuleState::Draining => snapshot.draining_to_replace,
1893                ModuleState::Starting
1894                | ModuleState::Running
1895                | ModuleState::Unresponsive
1896                | ModuleState::Stopped
1897                | ModuleState::Failed
1898                | ModuleState::Disabled => false,
1899            }
1900    }
1901}
1902
1903#[cfg(test)]
1904#[derive(Debug, Default)]
1905struct ReloadExitRecordGate {
1906    reached: tokio::sync::Notify,
1907    resume: tokio::sync::Notify,
1908}
1909
1910#[derive(Debug, Clone, Copy)]
1911enum RespawnKind {
1912    Spawn,
1913    Reload,
1914}
1915
1916#[derive(Debug, Clone, Copy)]
1917struct PendingRespawn {
1918    deadline: Instant,
1919    kind: RespawnKind,
1920    daemon_initiated: bool,
1921}
1922
1923type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1924
1925#[derive(Debug, Clone)]
1926struct SupervisorRuntimeConfig {
1927    /// Retained across an operation's deferred backoff, until its child starts.
1928    recovery_is_daemon: bool,
1929    start_requester: Option<Principal>,
1930    pending_activity_restart: bool,
1931    provenance_probe: ExecutableIdentityProbe,
1932    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1933    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1934    /// A reload acknowledges completion only after its replacement registers.
1935    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1936    restart_policy: RestartPolicy,
1937    /// This module's RESOLVED drain budget: per-module config when present,
1938    /// else `default_drain_timeout`.
1939    drain_timeout: Duration,
1940    /// Shared with the status handle so the attested value changes atomically
1941    /// when a rescan updates the running drain policy.
1942    effective_drain_timeout: Arc<Mutex<Duration>>,
1943    /// The supervisor-wide fallback, kept so a configuration update that
1944    /// REMOVES the per-module override can re-resolve to it.
1945    default_drain_timeout: Duration,
1946    health: HealthConfig,
1947    connection_file_path: Option<PathBuf>,
1948    capture_logs_dir: Option<PathBuf>,
1949    forwarding: Option<Arc<ForwardingTable>>,
1950    /// The shared handle, so every spawn path (initial, restart, reload) records the
1951    /// reserved-module launch nonce the HELLO verifier checks against.
1952    supervisor_handle: Option<SupervisorHandle>,
1953    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1954    /// status queries.
1955    ///
1956    /// One ring per module, held across every respawn. The lines explaining an exit
1957    /// are written BEFORE that exit, so a ring recreated per process would be empty
1958    /// exactly when it is asked for.
1959    stderr_ring: Arc<Mutex<StderrRing>>,
1960    terminal_ring: Arc<Mutex<TerminalRing>>,
1961    spawn_events: SpawnEventFeed,
1962    child_roster: ChildRoster,
1963    #[cfg(target_os = "linux")]
1964    cgroup_placement: Option<subc_cgroup::Placement>,
1965    #[cfg(test)]
1966    test_seed_stale_facts_before_enable_spawn: bool,
1967    #[cfg(test)]
1968    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1969}
1970
1971impl SupervisorRuntimeConfig {
1972    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1973        FailureContext {
1974            daemon_initiated,
1975            shutdown: self.child_roster.shutdown_flag(),
1976        }
1977    }
1978}
1979
1980#[derive(Debug, Clone, PartialEq, Eq)]
1981struct SupervisedConfiguration {
1982    spec: ModuleSpec,
1983    health: HealthConfig,
1984}
1985
1986/// Shared daemon lookup table for supervised module handles.
1987///
1988/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1989/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1990/// launch nonces recorded at spawn are checked by the same daemon instance.
1991#[derive(Debug, Clone, Default)]
1992pub struct SupervisorHandle {
1993    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1994    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1995    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1996    /// Module ids the supervisor has taken on. An id is added BEFORE the
1997    /// module's first process is spawned and removed only when the module
1998    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1999    /// the keys of `modules`.
2000    ///
2001    /// `modules` cannot answer "is this module configured?" on its own: a
2002    /// [`SupervisedModule`] only exists once its process has been spawned, and
2003    /// a fast child can connect, register, sync its scopes and ask about them
2004    /// before the supervisor has inserted it. Answering "not configured" in that
2005    /// gap makes scope admission refuse with the terminal "will never sync"
2006    /// instead of the retryable "has not synced yet".
2007    configured_ids: Arc<Mutex<HashSet<String>>>,
2008    spawn_events: SpawnEventFeed,
2009    /// The current expected launch nonce for each reserved module_id. Set when the
2010    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2011    /// non-reserved module never has an entry here and is never nonce-checked.
2012    /// Reserved module ids and the nonce that authorizes their next HELLO.
2013    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2014    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2015    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2016    /// had NO entry and admitted anyone: the reservation protected the nonce
2017    /// holder, not the NAME (found live by CKCRED's canary probe registering
2018    /// against a reserved scratch id).
2019    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2020    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2021    ///
2022    /// This is deliberately in-memory only: subc is state-free across daemon
2023    /// restarts, and the tombstone only explains the hours-after-removal window
2024    /// while this executing daemon is still alive. Do not persist it in a store.
2025    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2026    /// The current launch nonce for every supervised spawn. This is separate from
2027    /// reserved_nonces because consumer route.open attestation applies to all spawned
2028    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2029    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2030    /// Reserved namespace prefixes mapped to the supervised owner module whose
2031    /// current spawn nonce authorizes HELLO claims below the prefix.
2032    ///
2033    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2034    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2035    /// accidental collisions and lower-trust processes from squatting protected
2036    /// namespaces.
2037    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2038    /// Blue/green swaps in progress, by module id. An entry exists from just
2039    /// before the candidate process is spawned until the swap has failed, or
2040    /// has cut over and the old process is gone. While it exists, HELLO for the
2041    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2042    /// consumer attestation accepts both processes' nonces.
2043    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2044    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2045    promotion_observer: PromotionObserverSlot,
2046    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2047    /// this daemon-wide ordering, a rescan could retire or update a module while a
2048    /// concurrent reload still held its old handle and launch specification.
2049    operation_lock: Arc<AsyncMutex<()>>,
2050}
2051
2052/// Told when a swap has promoted its candidate to be the module's active
2053/// registration.
2054///
2055/// An ordinary HELLO runs the control plane's registration side effects (the
2056/// capability cache, the deny census, the requirement recompute) as it
2057/// registers. A swap candidate's HELLO does not, because it is not routable;
2058/// promotion is when those must run instead, and promotion happens in the
2059/// supervisor, which has no other way into the control handler.
2060pub(crate) trait SwapPromotionObserver: Send + Sync {
2061    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2062}
2063
2064/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2065/// control handler) owns this handle, so a strong reference back would be a
2066/// cycle that keeps both alive.
2067#[derive(Clone, Default)]
2068struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2069
2070impl fmt::Debug for PromotionObserverSlot {
2071    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2072        f.write_str("PromotionObserverSlot")
2073    }
2074}
2075
2076/// The nonces of one open swap.
2077#[derive(Debug, Clone)]
2078struct OpenSwap {
2079    /// The launch nonce minted for the candidate process. It is the swap
2080    /// token: the only thing that admits a HELLO into the candidate slot.
2081    candidate_nonce: String,
2082    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2083    /// here because cutover moves the module's recorded spawn nonce to the
2084    /// candidate while the incumbent is still draining and its consumers are
2085    /// still attesting with this one.
2086    incumbent_nonce: Option<String>,
2087    /// Set once a HELLO has been admitted with the swap token, so the token
2088    /// admits one registration and cannot be replayed after cutover empties
2089    /// the candidate slot.
2090    candidate_admitted: bool,
2091}
2092
2093/// What the swap gate says about a HELLO. See
2094/// [`SupervisorHandle::swap_hello_admission`].
2095#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2096pub(crate) enum SwapHelloAdmission {
2097    /// No swap is open for the id (or the HELLO carries the incumbent's own
2098    /// nonce); the ordinary gates decide.
2099    NotSwapping,
2100    /// The HELLO carries the swap token: register it into the candidate slot.
2101    Candidate,
2102    /// A swap is open and the HELLO carries a nonce the supervisor did not
2103    /// mint for this id, no nonce, or a token already used.
2104    Refused,
2105}
2106
2107#[derive(Debug, Clone, PartialEq, Eq)]
2108pub(crate) enum ReservedHelloRejection {
2109    Exact {
2110        module_id: String,
2111    },
2112    Prefix {
2113        prefix: String,
2114        owner_module_id: String,
2115    },
2116}
2117
2118impl SupervisorHandle {
2119    pub fn new() -> Self {
2120        Self::default()
2121    }
2122
2123    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2124        self.spawn_events.snapshot()
2125    }
2126
2127    pub(crate) fn subscribe_spawns(
2128        &self,
2129        connection_id: ConnectionId,
2130        corr: u64,
2131        version: u8,
2132        since: Option<SpawnCursor>,
2133        sink: FrameSink,
2134    ) -> Result<(), SpawnSubscribeRefusal> {
2135        self.spawn_events
2136            .subscribe(connection_id, corr, version, since, sink)
2137    }
2138
2139    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2140        self.spawn_events.cancel(connection_id, corr)
2141    }
2142
2143    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2144        self.spawn_events.remove_connection(connection_id);
2145    }
2146
2147    #[cfg(any(test, feature = "test-support"))]
2148    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2149        assert!(capacity > 0, "spawn event capacity must be non-zero");
2150        self.spawn_events.set_capacity(capacity);
2151    }
2152
2153    #[cfg(any(test, feature = "test-support"))]
2154    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2155        self.spawn_events.subscriber_count()
2156    }
2157
2158    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2159    /// a respawn invalidates stale consumer identities.
2160    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2161        self.spawn_nonces
2162            .lock()
2163            .unwrap_or_else(|poisoned| poisoned.into_inner())
2164            .insert(module_id.to_string(), nonce);
2165    }
2166
2167    /// Record the launch nonce expected from the next HELLO for a reserved module,
2168    /// replacing any prior nonce (a respawn invalidates the previous one).
2169    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2170        self.reserved_nonces
2171            .lock()
2172            .unwrap_or_else(|poisoned| poisoned.into_inner())
2173            .insert(module_id.to_string(), Some(nonce));
2174    }
2175
2176    /// Record namespace prefixes owned by a supervised module.
2177    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2178        let mut owners = self
2179            .reserved_prefix_owners
2180            .lock()
2181            .unwrap_or_else(|poisoned| poisoned.into_inner());
2182        owners.retain(|_, owner| owner != owner_module_id);
2183        for prefix in prefixes {
2184            owners.insert(prefix.clone(), owner_module_id.to_string());
2185        }
2186    }
2187
2188    /// The launch nonce most recently minted for a module's spawn, if any.
2189    #[cfg(test)]
2190    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2191        self.spawn_nonces
2192            .lock()
2193            .unwrap_or_else(|poisoned| poisoned.into_inner())
2194            .get(module_id)
2195            .cloned()
2196    }
2197
2198    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2199        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2200        let spawn_nonce = self
2201            .spawn_nonces
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .get(&spec.module_id)
2205            .cloned();
2206        let mut reserved_nonces = self
2207            .reserved_nonces
2208            .lock()
2209            .unwrap_or_else(|poisoned| poisoned.into_inner());
2210        if spec.reserved {
2211            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2212            // reserved name whose module has never spawned has no legitimate
2213            // holder, and the entry's absence is what used to leave the name
2214            // open to the first claimant.
2215            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2216        }
2217        drop(reserved_nonces);
2218        // A later unreserved declaration must not silently unreserve an id that
2219        // was retained after its reserved configuration was removed. The explicit
2220        // release ceremony is the only operation that retires that gate.
2221        self.removal_tombstones
2222            .lock()
2223            .unwrap_or_else(|poisoned| poisoned.into_inner())
2224            .remove(&spec.module_id);
2225    }
2226
2227    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2228    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2229    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2230    /// with no matching prefix are always authorized.
2231    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2232        self.reserved_hello_verification(module_id, presented)
2233            .is_ok()
2234    }
2235
2236    /// Whether HELLO passed a nonce check that was actually required by reserved
2237    /// module policy, or the refusal explaining why a required check failed.
2238    pub(crate) fn reserved_hello_verification(
2239        &self,
2240        module_id: &str,
2241        presented: Option<&str>,
2242    ) -> Result<bool, ReservedHelloRejection> {
2243        let nonces = self
2244            .reserved_nonces
2245            .lock()
2246            .unwrap_or_else(|poisoned| poisoned.into_inner());
2247        if let Some(expected) = nonces.get(module_id) {
2248            // `None` = reserved with no legitimate holder: refuse every
2249            // presentation, because no process can hold a nonce that was never
2250            // minted. Only a real minted nonce admits, in constant time.
2251            let authorized = match expected {
2252                Some(expected) => {
2253                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2254                }
2255                None => false,
2256            };
2257            if authorized {
2258                return Ok(true);
2259            }
2260            return Err(ReservedHelloRejection::Exact {
2261                module_id: module_id.to_string(),
2262            });
2263        }
2264        drop(nonces);
2265
2266        let matched_prefix = self
2267            .reserved_prefix_owners
2268            .lock()
2269            .unwrap_or_else(|poisoned| poisoned.into_inner())
2270            .iter()
2271            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2272            .max_by_key(|(prefix, _)| prefix.len())
2273            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2274        let Some((prefix, owner_module_id)) = matched_prefix else {
2275            return Ok(false);
2276        };
2277
2278        let authorized = presented.is_some_and(|presented| {
2279            self.spawn_nonces
2280                .lock()
2281                .unwrap_or_else(|poisoned| poisoned.into_inner())
2282                .get(&owner_module_id)
2283                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2284                // While the owner is being swapped, children started by
2285                // either of its two processes hold that process's nonce.
2286                || self.swap_nonce_matches(&owner_module_id, presented)
2287        });
2288        if authorized {
2289            Ok(true)
2290        } else {
2291            Err(ReservedHelloRejection::Prefix {
2292                prefix,
2293                owner_module_id,
2294            })
2295        }
2296    }
2297
2298    /// Whether a consumer connection proved it came from a daemon-spawned module.
2299    ///
2300    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2301    /// accepted only for module ids the supervisor has spawned.
2302    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2303        if presented.is_empty() {
2304            return false;
2305        }
2306        let nonces = self
2307            .spawn_nonces
2308            .lock()
2309            .unwrap_or_else(|poisoned| poisoned.into_inner());
2310        let current = nonces
2311            .get(module_id)
2312            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2313        drop(nonces);
2314        // During a swap two processes of the module are alive, and a consumer
2315        // started by either one presents that process's nonce. Accepting only
2316        // the recorded one would fail the incumbent's consumers for the whole
2317        // overlap once cutover moves the record to the candidate.
2318        current || self.swap_nonce_matches(module_id, presented)
2319    }
2320
2321    /// Whether `presented` is either nonce of an open swap for `module_id`.
2322    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2323        let swaps = self
2324            .swaps
2325            .lock()
2326            .unwrap_or_else(|poisoned| poisoned.into_inner());
2327        swaps.get(module_id).is_some_and(|swap| {
2328            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2329                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2330                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2331                })
2332        })
2333    }
2334
2335    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2336    /// Called before the candidate process exists.
2337    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2338        let incumbent_nonce = self
2339            .spawn_nonces
2340            .lock()
2341            .unwrap_or_else(|poisoned| poisoned.into_inner())
2342            .get(module_id)
2343            .cloned();
2344        self.swaps
2345            .lock()
2346            .unwrap_or_else(|poisoned| poisoned.into_inner())
2347            .insert(
2348                module_id.to_string(),
2349                OpenSwap {
2350                    candidate_nonce,
2351                    incumbent_nonce,
2352                    candidate_admitted: false,
2353                },
2354            );
2355    }
2356
2357    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2358    /// the module's recorded one.
2359    pub(crate) fn close_swap(&self, module_id: &str) {
2360        self.swaps
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner())
2363            .remove(module_id);
2364    }
2365
2366    /// Install the observer told about swap promotions, replacing any earlier
2367    /// one.
2368    pub(crate) fn set_swap_promotion_observer(
2369        &self,
2370        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2371    ) {
2372        *self
2373            .promotion_observer
2374            .0
2375            .lock()
2376            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2377    }
2378
2379    /// Tell the installed observer, if it is still alive, that a swap promoted
2380    /// `registration`.
2381    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2382        let observer = self
2383            .promotion_observer
2384            .0
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .as_ref()
2388            .and_then(std::sync::Weak::upgrade);
2389        if let Some(observer) = observer {
2390            observer.swap_promoted(registration);
2391        }
2392    }
2393
2394    /// Whether a swap is open for `module_id`.
2395    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2396        self.swaps
2397            .lock()
2398            .unwrap_or_else(|poisoned| poisoned.into_inner())
2399            .contains_key(module_id)
2400    }
2401
2402    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2403    /// respawn would, once cutover has made the candidate the module's process.
2404    /// The swap stays open so the incumbent's nonce keeps attesting until the
2405    /// incumbent has drained and exited.
2406    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2407        let candidate_nonce = self
2408            .swaps
2409            .lock()
2410            .unwrap_or_else(|poisoned| poisoned.into_inner())
2411            .get(module_id)
2412            .map(|swap| swap.candidate_nonce.clone());
2413        let Some(nonce) = candidate_nonce else {
2414            return;
2415        };
2416        self.set_spawn_nonce(module_id, nonce.clone());
2417        if reserved {
2418            self.set_reserved_nonce(module_id, nonce);
2419        }
2420    }
2421
2422    /// The swap gate for a HELLO claiming `module_id`.
2423    ///
2424    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2425    /// presents the candidate nonce, which the reserved gate (holding the
2426    /// incumbent's nonce) would refuse as `reserved_module` before swap
2427    /// admission was ever reached. And it applies to unreserved ids too: for an
2428    /// unreserved id the only thing that ever stopped a second process claiming
2429    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2430    /// refusal a swap lifts for its candidate.
2431    ///
2432    /// The incumbent's own nonce falls through to the ordinary gates, which
2433    /// treat it as they always have (a live incumbent is refused as a
2434    /// duplicate). Anything else while a swap is open is refused, including an
2435    /// absent nonce.
2436    pub(crate) fn swap_hello_admission(
2437        &self,
2438        module_id: &str,
2439        presented: Option<&str>,
2440    ) -> SwapHelloAdmission {
2441        let swaps = self
2442            .swaps
2443            .lock()
2444            .unwrap_or_else(|poisoned| poisoned.into_inner());
2445        let Some(swap) = swaps.get(module_id) else {
2446            return SwapHelloAdmission::NotSwapping;
2447        };
2448        let Some(presented) = presented else {
2449            return SwapHelloAdmission::Refused;
2450        };
2451        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2452            return if swap.candidate_admitted {
2453                SwapHelloAdmission::Refused
2454            } else {
2455                SwapHelloAdmission::Candidate
2456            };
2457        }
2458        if swap
2459            .incumbent_nonce
2460            .as_deref()
2461            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2462        {
2463            return SwapHelloAdmission::NotSwapping;
2464        }
2465        SwapHelloAdmission::Refused
2466    }
2467
2468    /// Record that the swap token has registered a candidate, so it admits no
2469    /// second HELLO.
2470    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2471        if let Some(swap) = self
2472            .swaps
2473            .lock()
2474            .unwrap_or_else(|poisoned| poisoned.into_inner())
2475            .get_mut(module_id)
2476        {
2477            swap.candidate_admitted = true;
2478        }
2479    }
2480
2481    /// Test/support lookup for the current launch nonce of a supervised spawn.
2482    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2483        self.spawn_nonces
2484            .lock()
2485            .unwrap_or_else(|poisoned| poisoned.into_inner())
2486            .get(module_id)
2487            .cloned()
2488    }
2489
2490    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2491    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2492        self.reserved_nonces
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner())
2495            .get(module_id)
2496            .cloned()
2497            .flatten()
2498    }
2499
2500    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2501        // Normally already marked before the process was spawned; marking here
2502        // too keeps `configured_ids` a superset of the roster for any caller
2503        // that inserts a module directly.
2504        self.mark_configured(module.module_id());
2505        let mut modules = self
2506            .modules
2507            .lock()
2508            .unwrap_or_else(|poisoned| poisoned.into_inner());
2509        modules.insert(module.module_id().to_string(), module)
2510    }
2511
2512    /// Record that the supervisor has taken on `module_id`. Called before the
2513    /// module's first process is spawned, so that by the time that process can
2514    /// register, [`Self::is_configured`] already answers true.
2515    fn mark_configured(&self, module_id: &str) {
2516        self.configured_ids
2517            .lock()
2518            .unwrap_or_else(|poisoned| poisoned.into_inner())
2519            .insert(module_id.to_string());
2520    }
2521
2522    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2523    /// before it was ever put on the roster. A module already on the roster
2524    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2525    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2526        let modules = self
2527            .modules
2528            .lock()
2529            .unwrap_or_else(|poisoned| poisoned.into_inner());
2530        if !modules.contains_key(module_id) {
2531            self.configured_ids
2532                .lock()
2533                .unwrap_or_else(|poisoned| poisoned.into_inner())
2534                .remove(module_id);
2535        }
2536    }
2537
2538    /// Whether `module_id` is a module this daemon supervises: on the roster,
2539    /// or about to be (its process is being spawned right now).
2540    ///
2541    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2542    /// for scopes: a supervised module's process can register and sync before
2543    /// [`Self::get`] can return it, and in that window it is still a module
2544    /// that will sync, not one that never will.
2545    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2546        self.configured_ids
2547            .lock()
2548            .unwrap_or_else(|poisoned| poisoned.into_inner())
2549            .contains(module_id)
2550    }
2551
2552    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2553        let modules = self
2554            .modules
2555            .lock()
2556            .unwrap_or_else(|poisoned| poisoned.into_inner());
2557        modules.get(module_id).cloned()
2558    }
2559
2560    pub(crate) fn record_late_health_answer(
2561        &self,
2562        module_id: &str,
2563        latency_ms: u64,
2564    ) -> Result<bool, SuperviseError> {
2565        let Some(module) = self.get(module_id) else {
2566            return Ok(false);
2567        };
2568        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2569            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2570            state.health.last_late_answer_latency_ms = Some(latency_ms);
2571            // A late answer is an answer: the module served the probe, just past
2572            // the deadline. Leaving the miss streak in place while logging
2573            // "proves the module is alive" is how a CPU-starved module that
2574            // answers every probe a few seconds late still marches to the
2575            // threshold and gets killed — the exact kill class `NoAnswer` is
2576            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2577            // is degradation, and degradation reports; it does not restart.
2578            state.health.consecutive_failures = 0;
2579        })?;
2580        Ok(true)
2581    }
2582
2583    /// Arm the one-shot marker for the module process that this caller
2584    /// deliberately initiated severance against. Generic connection teardown
2585    /// must not call this:
2586    /// a surviving process would otherwise retain an exemption for a later
2587    /// genuine crash.
2588    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2589        let Some(module) = self.get(module_id) else {
2590            return Ok(false);
2591        };
2592        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2593        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2594            return Ok(false);
2595        };
2596        drop(snapshot);
2597        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2598    }
2599
2600    pub fn list(&self) -> Vec<SupervisedModule> {
2601        let modules = self
2602            .modules
2603            .lock()
2604            .unwrap_or_else(|poisoned| poisoned.into_inner());
2605        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2606        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2607        modules
2608    }
2609
2610    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2611        self.activity_images
2612            .lock()
2613            .unwrap_or_else(|p| p.into_inner())
2614            .remove(module_id);
2615        self.spawn_nonces
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        self.close_swap(module_id);
2620        let mut reserved_nonces = self
2621            .reserved_nonces
2622            .lock()
2623            .unwrap_or_else(|poisoned| poisoned.into_inner());
2624        if reserved_nonces.contains_key(module_id) {
2625            // The old nonce must die with the removed process, but the exact-id
2626            // gate remains until an operator explicitly releases it.
2627            reserved_nonces.insert(module_id.to_string(), None);
2628        }
2629        drop(reserved_nonces);
2630        self.reserved_prefix_owners
2631            .lock()
2632            .unwrap_or_else(|poisoned| poisoned.into_inner())
2633            .retain(|_, owner| owner != module_id);
2634        let removed = self
2635            .modules
2636            .lock()
2637            .unwrap_or_else(|poisoned| poisoned.into_inner())
2638            .remove(module_id);
2639        self.configured_ids
2640            .lock()
2641            .unwrap_or_else(|poisoned| poisoned.into_inner())
2642            .remove(module_id);
2643        removed
2644    }
2645
2646    /// Remember a module removed by a non-preview rescan so route.open can
2647    /// distinguish that intentional removal from an unknown id.
2648    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2649        self.removal_tombstones
2650            .lock()
2651            .unwrap_or_else(|poisoned| poisoned.into_inner())
2652            .insert(module_id.to_string(), unix_ms_now());
2653    }
2654
2655    /// Return how long ago a rescan removed this module in milliseconds.
2656    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2657        self.removal_tombstones
2658            .lock()
2659            .unwrap_or_else(|poisoned| poisoned.into_inner())
2660            .get(module_id)
2661            .copied()
2662            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2663    }
2664
2665    /// Retire a reserved-id gate only after its module has left supervision.
2666    ///
2667    /// A retained gate has no live nonce (`None`), so releasing any other entry
2668    /// would weaken a currently configured or otherwise active reservation.
2669    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2670        if self.get(module_id).is_some() {
2671            return false;
2672        }
2673        let mut reserved_nonces = self
2674            .reserved_nonces
2675            .lock()
2676            .unwrap_or_else(|poisoned| poisoned.into_inner());
2677        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2678            return false;
2679        }
2680        reserved_nonces.remove(module_id);
2681        true
2682    }
2683
2684    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2685        Arc::clone(&self.operation_lock)
2686    }
2687}
2688
2689/// Process supervisor for subc-owned singleton modules.
2690#[derive(Debug, Clone)]
2691pub struct Supervisor {
2692    registry: Arc<Registry>,
2693    restart_policy: RestartPolicy,
2694    drain_timeout: Duration,
2695    connection_file_path: Option<PathBuf>,
2696    capture_logs_dir: Option<PathBuf>,
2697    forwarding: Option<Arc<ForwardingTable>>,
2698    process_liveness: Arc<SupervisorProcessLiveness>,
2699    supervisor_handle: Option<SupervisorHandle>,
2700    health: HealthConfig,
2701    daemon_start_clock: crate::clock::StartClock,
2702    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2703    spawn_events: SpawnEventFeed,
2704    provenance_probe: ExecutableIdentityProbe,
2705    /// Every process spawned through this supervisor (and its clones) and not
2706    /// yet reaped, so daemon shutdown can end them.
2707    child_roster: ChildRoster,
2708    #[cfg(target_os = "linux")]
2709    cgroup_placement: Option<subc_cgroup::Placement>,
2710    #[cfg(test)]
2711    test_after_first_spawn: AfterFirstSpawnHook,
2712}
2713
2714/// A roster entry whose monitor and first process have not started yet.
2715/// Keeping the command receiver here lets discovery serve status immediately
2716/// while start/stop requests wait until bootstrap has published the connection file.
2717pub(crate) struct RegisteredConfiguredModule {
2718    spec: ModuleSpec,
2719    runtime: SupervisorRuntimeConfig,
2720    module: SupervisedModule,
2721    commands: mpsc::Receiver<SupervisorCommand>,
2722}
2723
2724/// Test-only hook run on the path that takes on a new module, right after its
2725/// first `spawn_child` returns (the process exists and could already be
2726/// registering) and before that process is handed to the module's supervise
2727/// loop and put on the roster. Lets a test observe what a fast child would see
2728/// in that window without racing a real one.
2729#[cfg(test)]
2730#[derive(Clone, Default)]
2731struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2732
2733#[cfg(test)]
2734type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2735
2736#[cfg(test)]
2737impl fmt::Debug for AfterFirstSpawnHook {
2738    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2739        f.write_str("AfterFirstSpawnHook")
2740    }
2741}
2742
2743#[cfg(test)]
2744impl AfterFirstSpawnHook {
2745    fn run(&self, module_id: &str) {
2746        if let Some(hook) = &self.0 {
2747            hook(module_id);
2748        }
2749    }
2750}
2751
2752impl Supervisor {
2753    #[cfg(test)]
2754    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2755        let supervisor = Self::new(registry, policy);
2756        #[cfg(target_os = "macos")]
2757        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2758        supervisor
2759    }
2760    /// Verify the trampoline once when configured. Missing private OS support
2761    /// refuses every macOS launch by name but does not stop the daemon's control
2762    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2763    /// entry point; the library must not exec an arbitrary hosting program.
2764    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2765        let path = path.into();
2766        #[cfg(target_os = "macos")]
2767        {
2768            let result = probe_privacy_trampoline(&path).map(|()| path);
2769            if let Err(cause) = &result {
2770                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2771            }
2772            self.child_roster.set_privacy_trampoline(result);
2773        }
2774        #[cfg(not(target_os = "macos"))]
2775        let _ = path;
2776        self
2777    }
2778    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2779    /// capability-requirement log to read.
2780    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2781        self.child_roster.shutdown_flag()
2782    }
2783
2784    /// The first step of an announced daemon shutdown, before the notice and
2785    /// before any connection is closed.
2786    ///
2787    /// Sets the daemon-shutdown flag first: from here on no module is
2788    /// respawned (crash restart, operator restart, or swap), and every child
2789    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2790    /// the module exits on the EOF this shutdown gives it or is signalled by a
2791    /// service manager that kills the whole cgroup. Then writes the journal's
2792    /// shutdown marker, which records the instant and closes this daemon
2793    /// incarnation's stretch of the journal.
2794    #[cfg(unix)]
2795    pub(crate) fn begin_daemon_shutdown(&self) {
2796        self.child_roster.close();
2797        if let Some(journal) = &self.terminal_journal {
2798            journal.stamp_shutdown();
2799        }
2800    }
2801
2802    /// Announce a cut while established connections can still carry replies.
2803    /// These budgets promise notice and a bounded wait, not child completion;
2804    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2805    #[cfg(unix)]
2806    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2807        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2808        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2809        let Some(forwarding) = &self.forwarding else {
2810            return Ok(());
2811        };
2812        let notice_started = Instant::now();
2813        let modules = forwarding
2814            .begin_daemon_drain()
2815            .map_err(SuperviseError::Forwarding)?;
2816        let deadline_ms =
2817            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2818        let mut notices = tokio::task::JoinSet::new();
2819        let mut drains = Vec::new();
2820        for (module_id, endpoint) in modules {
2821            let Some(target) = forwarding
2822                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2823                .map_err(SuperviseError::Forwarding)?
2824            else {
2825                continue;
2826            };
2827            let routes = forwarding
2828                .endpoint_routes(target.endpoint)
2829                .map_err(SuperviseError::Forwarding)?;
2830            // Restart allows deployed consumers to reopen after the new daemon
2831            // appears. The wire reason stays `restart`; what tells a daemon cut
2832            // apart from a module restart afterwards is the terminal record
2833            // itself, whose disposition is `daemon_shutdown` for every exit
2834            // observed once `begin_daemon_shutdown` has run.
2835            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2836                reason: RouteCloseReason::Restart,
2837                deadline_ms,
2838            })
2839            .expect("module draining serializes");
2840            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2841            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2842            for route in routes {
2843                let client = route.goodbye_target;
2844                if let Some((_, channels)) = clients
2845                    .iter_mut()
2846                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2847                {
2848                    channels.push(client.channel);
2849                } else {
2850                    let channel = client.channel;
2851                    clients.push((client, vec![channel]));
2852                }
2853            }
2854            for (client, mut channels) in clients {
2855                channels.sort_unstable();
2856                channels.dedup();
2857                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2858                    module_id: module_id.clone(),
2859                    channels,
2860                    reason: RouteCloseReason::Restart,
2861                })
2862                .expect("route closing serializes");
2863                recipients.push((client.sink, client.negotiated_ver, closing));
2864            }
2865            for (sink, version, body) in recipients {
2866                notices.spawn(async move {
2867                    let frame = Frame::build_with_version(
2868                        version,
2869                        FrameType::Push,
2870                        control_flags(),
2871                        0,
2872                        0,
2873                        0,
2874                        body,
2875                    )
2876                    .expect("bounded lifecycle notice frame builds");
2877                    sink.send_flushed(frame).await
2878                });
2879            }
2880            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2881            drains.push((module_id, target.endpoint, gauges));
2882        }
2883        // A quiet forwarding table is not proof that queued notices reached the
2884        // socket. Wait for writer flush acknowledgements before testing quiescence.
2885        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2886        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2887            if !matches!(result, Ok(Ok(()))) {
2888                warn!(?result, "daemon shutdown notice delivery failed");
2889            }
2890        }
2891        notices.abort_all();
2892        let notice_elapsed = notice_started.elapsed();
2893        let drain_started = Instant::now();
2894        let deadline = drain_started + DRAIN_BUDGET;
2895        let mut waits = tokio::task::JoinSet::new();
2896        for (module_id, endpoint, gauges) in drains {
2897            let forwarding = Arc::clone(forwarding);
2898            let mut runtime = self.runtime_config();
2899            runtime.health.cadence = Duration::from_millis(100);
2900            waits.spawn(async move {
2901                let wait_result = wait_for_forwarding_quiescence(
2902                    &forwarding,
2903                    &module_id,
2904                    &runtime,
2905                    endpoint,
2906                    deadline,
2907                    &gauges,
2908                    DrainScope::Endpoint(endpoint),
2909                    &mut DrainGaugeTrace::new(&gauges),
2910                )
2911                .await;
2912                // Store the measurement before the task returns: quiescence
2913                // may be reached just as the deadline stops the join loop from
2914                // collecting results. Unfinished or failed waits report false.
2915                forwarding
2916                    .record_daemon_drain_result(
2917                        endpoint,
2918                        drained_after_quiescence_wait(&wait_result),
2919                    )
2920                    .map_err(SuperviseError::Forwarding)?;
2921                wait_result
2922            });
2923        }
2924        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2925            if !matches!(result, Ok(Ok(true))) {
2926                warn!(?result, "daemon shutdown drain did not reach quiescence");
2927            }
2928        }
2929        // Measure only the forwarding drain, not child teardown or the caller's
2930        // signal delivery and process-reaping delays.
2931        info!(
2932            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2933            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2934            "daemon shutdown forwarding drain complete"
2935        );
2936        Ok(())
2937    }
2938
2939    /// The last step of an announced daemon shutdown, after the notice and the
2940    /// drain: send every registered module a module GOODBYE, the same planned
2941    /// stop signal `ck module stop` gives, then close every connection so each
2942    /// subc module sees EOF and starts its own teardown, then end every
2943    /// supervised child that has not exited
2944    /// by its own deadline (its drain budget, capped). Modules lead their own
2945    /// process groups, so a
2946    /// service manager's group kill no longer reaches them; without this a
2947    /// child that does not stop on EOF (every `protocol: "none"` child, which
2948    /// has no connection) would outlive the daemon. Every wait is bounded (see
2949    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2950    #[cfg(unix)]
2951    pub(crate) async fn end_children_for_daemon_shutdown(
2952        &self,
2953        already_escalated: bool,
2954        escalate: impl std::future::Future<Output = ()>,
2955    ) {
2956        tokio::pin!(escalate);
2957        let mut escalated = already_escalated;
2958        if let Some(forwarding) = &self.forwarding {
2959            let reason = CloseReason::new(
2960                "daemon_shutdown",
2961                "the daemon is exiting after its shutdown notice and drain",
2962            );
2963            if escalated {
2964                // The operator asked to stop waiting: queue the GOODBYEs but
2965                // do not wait for them to be written.
2966                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2967            } else {
2968                tokio::select! {
2969                    biased;
2970                    _ = escalate.as_mut() => {
2971                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2972                        escalated = true;
2973                    }
2974                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2975                }
2976            }
2977            let closed = forwarding.close_all_connections(&reason);
2978            debug!(closed, "closed established connections for daemon shutdown");
2979        }
2980        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2981        // already completed and must not be polled again; the child shutdown
2982        // wait is told it is escalated and gets a future that never fires.
2983        let escalated_here = escalated && !already_escalated;
2984        let remaining_escalate = async move {
2985            if escalated_here {
2986                std::future::pending::<()>().await;
2987            } else {
2988                escalate.await;
2989            }
2990        };
2991        crate::child_roster::end_children_for_daemon_shutdown(
2992            &self.child_roster,
2993            escalated,
2994            remaining_escalate,
2995        )
2996        .await;
2997    }
2998
2999    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3000        Self {
3001            registry,
3002            restart_policy,
3003            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3004            connection_file_path: None,
3005            capture_logs_dir: None,
3006            forwarding: None,
3007            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3008            supervisor_handle: None,
3009            health: HealthConfig::default(),
3010            daemon_start_clock: crate::clock::StartClock::capture(),
3011            terminal_journal: None,
3012            spawn_events: SpawnEventFeed::default(),
3013            provenance_probe: ExecutableIdentityProbe::default(),
3014            child_roster: ChildRoster::default(),
3015            #[cfg(target_os = "linux")]
3016            cgroup_placement: None,
3017            #[cfg(test)]
3018            test_after_first_spawn: AfterFirstSpawnHook::default(),
3019        }
3020    }
3021
3022    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3023        self.drain_timeout = drain_timeout;
3024        self
3025    }
3026
3027    /// The drain budget a module gets when its config entry sets none.
3028    pub(crate) fn default_drain_timeout(&self) -> Duration {
3029        self.drain_timeout
3030    }
3031
3032    pub fn with_process_liveness(
3033        mut self,
3034        process_liveness: Arc<SupervisorProcessLiveness>,
3035    ) -> Self {
3036        self.process_liveness = process_liveness;
3037        self
3038    }
3039
3040    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3041        self.connection_file_path = Some(connection_file_path.into());
3042        self
3043    }
3044
3045    /// Enables daemon-owned capture files for supervised stdout and stderr.
3046    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3047        self.capture_logs_dir = Some(logs_dir.into());
3048        self
3049    }
3050
3051    /// Names this daemon lifetime in spawn events, independently of whether a
3052    /// terminal journal is configured.
3053    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3054        // A millisecond start stamp can repeat after clock rollback or a rapid
3055        // restart. Use the connection file's random daemon_id instead: it already
3056        // identifies this daemon lifetime independently of the wall clock.
3057        self.spawn_events.configure_incarnation(daemon_incarnation);
3058        self
3059    }
3060
3061    /// Enables best-effort history shared by every supervised module. Without
3062    /// it, terminal history is kept only in each module's in-memory ring.
3063    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3064        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3065        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3066            path,
3067            daemon_incarnation,
3068        )));
3069        this
3070    }
3071
3072    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3073        self.forwarding = Some(forwarding);
3074        self
3075    }
3076
3077    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3078        self.spawn_events = supervisor_handle.spawn_events.clone();
3079        self.supervisor_handle = Some(supervisor_handle);
3080        self
3081    }
3082
3083    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3084        self.health = health;
3085        self
3086    }
3087
3088    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3089    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3090    /// record is kept.
3091    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3092        self.child_roster.record_to(path.into());
3093        self
3094    }
3095
3096    #[cfg(target_os = "linux")]
3097    pub fn with_cgroup_placement(
3098        mut self,
3099        cgroup_placement: Option<subc_cgroup::Placement>,
3100    ) -> Self {
3101        self.cgroup_placement = cgroup_placement;
3102        self
3103    }
3104
3105    /// Spawn `spec.program` and start monitoring it.
3106    ///
3107    /// The child is expected to parse `--subc <connection-file-path>`, read the
3108    /// TCP+key connection file, authenticate to the already-running listener, and
3109    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3110    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3111        validate_spec(&spec)?;
3112        self.establish_identity(&spec);
3113
3114        let runtime = self.runtime_config();
3115        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3116        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3117        let spawned = spawn_child(
3118            &spec,
3119            runtime.connection_file_path.as_deref(),
3120            self.supervisor_handle.as_ref(),
3121            &runtime.stderr_ring,
3122            runtime.capture_logs_dir.as_deref(),
3123            &runtime.child_roster,
3124            #[cfg(target_os = "linux")]
3125            runtime.cgroup_placement.as_ref(),
3126        );
3127        #[cfg(test)]
3128        self.test_after_first_spawn.run(&spec.module_id);
3129        let child = match spawned {
3130            Ok(child) => child,
3131            Err(err) => {
3132                // Unlike the configured paths, a failed `spawn` leaves nothing
3133                // on the roster, so the module must not stay marked configured.
3134                self.abandon_unrostered(&spec.module_id);
3135                return Err(err);
3136            }
3137        };
3138        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3139        if let Some(spawn) = activity_spawn {
3140            spawn.complete(&snapshot);
3141        }
3142
3143        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3144    }
3145
3146    /// Make `spec`'s module count as configured, with its identity gates
3147    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3148    /// exists.
3149    ///
3150    /// Every path that takes on a new module calls this before `spawn_child`.
3151    /// The order is the point: the child can connect, register, sync its
3152    /// scopes and ask about them as soon as it is spawned, and the module is
3153    /// only put on the roster after `spawn_child` returns. Were the mark set
3154    /// with the roster entry, a fast child would see its own owner reported
3155    /// as not configured, and a scoped `route.open` in that window would be
3156    /// refused as terminal `scope_not_live` ("will never sync") instead of
3157    /// retryable `scope_not_synced`.
3158    fn establish_identity(&self, spec: &ModuleSpec) {
3159        if let Some(supervisor_handle) = &self.supervisor_handle {
3160            supervisor_handle.apply_identity_configuration(spec);
3161            supervisor_handle.mark_configured(&spec.module_id);
3162        }
3163    }
3164
3165    /// Take back [`Self::establish_identity`]'s configured mark when the
3166    /// module will not be put on the roster after all.
3167    fn abandon_unrostered(&self, module_id: &str) {
3168        if let Some(supervisor_handle) = &self.supervisor_handle {
3169            supervisor_handle.unmark_configured_unless_rostered(module_id);
3170        }
3171    }
3172
3173    /// Record a freshly spawned first process as running. On failure the
3174    /// module never reaches the roster, so its configured mark is taken back.
3175    fn mark_first_process_running(
3176        &self,
3177        spec: &ModuleSpec,
3178        runtime: &SupervisorRuntimeConfig,
3179        snapshot: &SharedSnapshot,
3180        child: &SupervisedChild,
3181    ) -> Result<(), SuperviseError> {
3182        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3183            self.abandon_unrostered(&spec.module_id);
3184            return Err(err);
3185        }
3186        self.process_liveness
3187            .track(spec.module_id.clone(), Arc::clone(snapshot));
3188        Ok(())
3189    }
3190
3191    /// Start supervising a module declared in daemon configuration.
3192    ///
3193    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3194    /// failures in the supervisor handle so operator-facing `supervisor.list`
3195    /// reflects every configured module while daemon startup continues.
3196    pub fn supervise_configured(
3197        &self,
3198        spec: ModuleSpec,
3199        enabled: bool,
3200    ) -> Result<SupervisedModule, SuperviseError> {
3201        self.supervise_configured_requested(
3202            spec,
3203            enabled,
3204            self.health.clone(),
3205            None,
3206            self.restart_policy,
3207            None,
3208        )
3209    }
3210
3211    /// Supervise a configured module with its own health, drain, and crash
3212    /// budget. The restart policy is per-module because the config file is:
3213    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3214    /// module that is expensive to restart should not be forced onto the same
3215    /// budget as one that is cheap.
3216    pub fn supervise_configured_with_health(
3217        &self,
3218        spec: ModuleSpec,
3219        enabled: bool,
3220        health: HealthConfig,
3221        drain_timeout_ms: Option<u64>,
3222        restart_policy: RestartPolicy,
3223    ) -> Result<SupervisedModule, SuperviseError> {
3224        self.supervise_configured_requested(
3225            spec,
3226            enabled,
3227            health,
3228            drain_timeout_ms,
3229            restart_policy,
3230            None,
3231        )
3232    }
3233
3234    pub(crate) fn supervise_configured_requested(
3235        &self,
3236        spec: ModuleSpec,
3237        enabled: bool,
3238        health: HealthConfig,
3239        drain_timeout_ms: Option<u64>,
3240        restart_policy: RestartPolicy,
3241        requester: Option<Principal>,
3242    ) -> Result<SupervisedModule, SuperviseError> {
3243        let registered = self.register_configured_with_health(
3244            spec,
3245            enabled,
3246            health,
3247            drain_timeout_ms,
3248            restart_policy,
3249        )?;
3250        let daemon_initiated = requester.is_none();
3251        self.start_registered_configured(registered, requester, daemon_initiated)
3252    }
3253
3254    /// Record a configured module as starting or disabled, without launching a
3255    /// process or running its monitor. Bootstrap registers the entire roster
3256    /// before publishing discovery, then activates it once children can read
3257    /// the connection file passed in their `--subc` argument.
3258    pub(crate) fn register_configured_with_health(
3259        &self,
3260        spec: ModuleSpec,
3261        enabled: bool,
3262        health: HealthConfig,
3263        drain_timeout_ms: Option<u64>,
3264        restart_policy: RestartPolicy,
3265    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3266        validate_spec(&spec)?;
3267        self.establish_identity(&spec);
3268
3269        let mut runtime = self.runtime_config();
3270        runtime.health = health;
3271        runtime.restart_policy = restart_policy;
3272        if let Some(ms) = drain_timeout_ms {
3273            runtime.drain_timeout = Duration::from_millis(ms);
3274            *runtime
3275                .effective_drain_timeout
3276                .lock()
3277                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3278        }
3279        let snapshot = if enabled {
3280            SupervisorSnapshot::starting()
3281        } else {
3282            SupervisorSnapshot::disabled()
3283        };
3284        let snapshot = {
3285            let mut snapshot = snapshot;
3286            if enabled {
3287                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3288                {
3289                    snapshot.waiting_for_previous_pid = Some(pid);
3290                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3291                }
3292            }
3293            snapshot
3294        };
3295        let (module, commands) =
3296            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3297        Ok(RegisteredConfiguredModule {
3298            spec,
3299            runtime,
3300            module,
3301            commands,
3302        })
3303    }
3304
3305    /// Activate a previously registered module. Queued control requests are
3306    /// served by the monitor after the initial configured spawn has settled.
3307    pub(crate) fn start_registered_configured(
3308        &self,
3309        registered: RegisteredConfiguredModule,
3310        requester: Option<Principal>,
3311        daemon_initiated: bool,
3312    ) -> Result<SupervisedModule, SuperviseError> {
3313        let RegisteredConfiguredModule {
3314            spec,
3315            mut runtime,
3316            module,
3317            commands,
3318        } = registered;
3319        runtime.start_requester = requester;
3320        runtime.recovery_is_daemon = daemon_initiated;
3321        let snapshot = &module.inner.snapshot;
3322        let should_spawn = {
3323            let state = lock_snapshot(snapshot)?;
3324            state.enabled && state.waiting_for_previous_pid.is_none()
3325        };
3326        let child = if should_spawn {
3327            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3328            let spawned = spawn_child(
3329                &spec,
3330                runtime.connection_file_path.as_deref(),
3331                self.supervisor_handle.as_ref(),
3332                &runtime.stderr_ring,
3333                runtime.capture_logs_dir.as_deref(),
3334                &runtime.child_roster,
3335                #[cfg(target_os = "linux")]
3336                runtime.cgroup_placement.as_ref(),
3337            );
3338            #[cfg(test)]
3339            self.test_after_first_spawn.run(&spec.module_id);
3340            match spawned {
3341                Ok(child) => {
3342                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3343                    if let Some(spawn) = activity_spawn {
3344                        spawn.complete(snapshot);
3345                    }
3346                    Some(child)
3347                }
3348                Err(err) => {
3349                    if runtime.health.critical {
3350                        error!(
3351                            module_id = %spec.module_id,
3352                            program = %spec.program.display(),
3353                            error = %err,
3354                            "critical configured module failed to spawn; marking failed and alerting"
3355                        );
3356                    } else {
3357                        error!(
3358                            module_id = %spec.module_id,
3359                            program = %spec.program.display(),
3360                            error = %err,
3361                            "configured module failed to spawn; marking failed and continuing"
3362                        );
3363                    }
3364                    fail_snapshot(
3365                        snapshot,
3366                        Some(&spec.module_id),
3367                        None,
3368                        &runtime.failure_context(daemon_initiated),
3369                    );
3370                    None
3371                }
3372            }
3373        } else {
3374            None
3375        };
3376        self.start_monitor(spec, runtime, &module, child, commands);
3377        Ok(module)
3378    }
3379
3380    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3381        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3382        SupervisorRuntimeConfig {
3383            recovery_is_daemon: true,
3384            start_requester: None,
3385            pending_activity_restart: false,
3386            provenance_probe: self.provenance_probe.clone(),
3387            scheduled_respawn: Arc::default(),
3388            deferred_reload_reply: Arc::default(),
3389            restart_policy: self.restart_policy,
3390            drain_timeout: self.drain_timeout,
3391            // Shared with this module's roster copy: daemon shutdown waits on
3392            // each child for the module's own drain budget, as resolved now.
3393            child_roster: self
3394                .child_roster
3395                .for_module(Arc::clone(&effective_drain_timeout)),
3396            effective_drain_timeout,
3397            default_drain_timeout: self.drain_timeout,
3398            health: self.health.clone(),
3399            connection_file_path: self.connection_file_path.clone(),
3400            capture_logs_dir: self.capture_logs_dir.clone(),
3401            forwarding: self.forwarding.clone(),
3402            supervisor_handle: self.supervisor_handle.clone(),
3403            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3404            terminal_ring: Arc::new(Mutex::new(
3405                TerminalRing::new(
3406                    TerminalRingConfig::default(),
3407                    self.daemon_start_clock.started_at_ms(),
3408                )
3409                .with_start_clock(self.daemon_start_clock)
3410                .with_journal(self.terminal_journal.clone())
3411                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3412            )),
3413            spawn_events: self.spawn_events.clone(),
3414            #[cfg(target_os = "linux")]
3415            cgroup_placement: self.cgroup_placement.clone(),
3416            #[cfg(test)]
3417            test_seed_stale_facts_before_enable_spawn: false,
3418            #[cfg(test)]
3419            test_reload_exit_record_gate: None,
3420        }
3421    }
3422
3423    fn supervised_module(
3424        &self,
3425        spec: ModuleSpec,
3426        runtime: SupervisorRuntimeConfig,
3427        snapshot: SharedSnapshot,
3428        child: Option<SupervisedChild>,
3429    ) -> SupervisedModule {
3430        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3431        self.start_monitor(spec, runtime, &module, child, commands);
3432        module
3433    }
3434
3435    fn register_module(
3436        &self,
3437        spec: &ModuleSpec,
3438        runtime: &SupervisorRuntimeConfig,
3439        snapshot: SharedSnapshot,
3440    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3441        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3442            spec: spec.clone(),
3443            health: runtime.health.clone(),
3444        }));
3445        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3446        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3447        // The module's OWN policy, which may be its per-module config rather than
3448        // the supervisor-wide one; status must report the budget the supervise
3449        // loop actually enforces.
3450        let restart_policy = runtime.restart_policy;
3451        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3452        let (tx, rx) = mpsc::channel(4);
3453        let module_id = spec.module_id.clone();
3454        let module = SupervisedModule {
3455            inner: Arc::new(SupervisedModuleInner {
3456                module_id: module_id.clone(),
3457                registry: Arc::clone(&self.registry),
3458                snapshot,
3459                configuration,
3460                stderr_ring,
3461                terminal_ring,
3462                commands: tx,
3463                monitor: Mutex::new(None),
3464                restart_policy,
3465                effective_drain_timeout,
3466                provenance_probe: self.provenance_probe.clone(),
3467            }),
3468        };
3469        // Identity gates and the configured mark must precede the roster entry,
3470        // so a client never sees a module without its admission policy.
3471        if let Some(supervisor_handle) = &self.supervisor_handle {
3472            supervisor_handle.insert(module.clone());
3473        }
3474        (module, rx)
3475    }
3476
3477    fn start_monitor(
3478        &self,
3479        spec: ModuleSpec,
3480        runtime: SupervisorRuntimeConfig,
3481        module: &SupervisedModule,
3482        child: Option<SupervisedChild>,
3483        commands: mpsc::Receiver<SupervisorCommand>,
3484    ) {
3485        let monitor = tokio::spawn(supervise_loop(
3486            spec,
3487            runtime,
3488            Arc::clone(&self.registry),
3489            Arc::clone(&self.process_liveness),
3490            Arc::clone(&module.inner.snapshot),
3491            child,
3492            commands,
3493        ));
3494        *module
3495            .inner
3496            .monitor
3497            .lock()
3498            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3499    }
3500}
3501
3502impl Default for Supervisor {
3503    fn default() -> Self {
3504        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3505    }
3506}
3507
3508/// Handle to one supervised child process.
3509#[derive(Clone)]
3510pub struct SupervisedModule {
3511    inner: Arc<SupervisedModuleInner>,
3512}
3513
3514struct SupervisedModuleInner {
3515    module_id: String,
3516    registry: Arc<Registry>,
3517    snapshot: SharedSnapshot,
3518    configuration: Arc<Mutex<SupervisedConfiguration>>,
3519    stderr_ring: Arc<Mutex<StderrRing>>,
3520    terminal_ring: Arc<Mutex<TerminalRing>>,
3521    commands: mpsc::Sender<SupervisorCommand>,
3522    monitor: Mutex<Option<JoinHandle<()>>>,
3523    /// Copied from the supervisor's runtime config at spawn so `status()` can
3524    /// report the restart budget without reaching back into the supervisor. The
3525    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3526    restart_policy: RestartPolicy,
3527    effective_drain_timeout: Arc<Mutex<Duration>>,
3528    provenance_probe: ExecutableIdentityProbe,
3529}
3530
3531impl fmt::Debug for SupervisedModule {
3532    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3533        f.debug_struct("SupervisedModule")
3534            .field("module_id", &self.inner.module_id)
3535            .field("status", &self.status())
3536            .finish_non_exhaustive()
3537    }
3538}
3539
3540impl SupervisedModule {
3541    pub fn module_id(&self) -> &str {
3542        &self.inner.module_id
3543    }
3544
3545    /// Test-only: put one probe miss on the streak, the way
3546    /// `handle_health_probe_failure` does, so tests can assert what a later
3547    /// event does to the streak without driving the whole probe loop.
3548    #[cfg(test)]
3549    pub(crate) fn record_health_probe_failure_for_test(
3550        &self,
3551        detail: &str,
3552    ) -> Result<(), SuperviseError> {
3553        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3554            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3555            state.health.detail = Some(detail.to_string());
3556        })
3557    }
3558
3559    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3560        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3561    }
3562
3563    pub(crate) fn registration_end_reason(
3564        &self,
3565    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3566        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3567        Ok(match snapshot.state {
3568            ModuleState::Draining if snapshot.draining_to_replace => {
3569                Some(RegistrationEndReason::SupervisorRestart)
3570            }
3571            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3572            ModuleState::Restarting if snapshot.draining_to_replace => {
3573                Some(RegistrationEndReason::SupervisorRestart)
3574            }
3575            _ => None,
3576        })
3577    }
3578
3579    /// The module's retained stderr, newest lines last.
3580    ///
3581    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3582    /// module, `supervisor.list` renders every module, and putting it in the
3583    /// shared snapshot would make each status read carry a payload almost nobody
3584    /// asked for. Callers that want the text ask for it.
3585    pub fn stderr_tail(
3586        &self,
3587        max_lines: Option<usize>,
3588        max_bytes: Option<usize>,
3589    ) -> StderrTailSnapshot {
3590        self.inner
3591            .stderr_ring
3592            .lock()
3593            .unwrap_or_else(|poisoned| poisoned.into_inner())
3594            .snapshot(max_lines, max_bytes)
3595    }
3596
3597    /// The module's bounded terminal history, oldest retained exit first.
3598    ///
3599    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3600    /// daemon whose in-memory history was necessarily reset.
3601    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3602        self.inner
3603            .terminal_ring
3604            .lock()
3605            .unwrap_or_else(|poisoned| poisoned.into_inner())
3606            .snapshot()
3607    }
3608
3609    /// Retained observations from the current ring and all journal generations.
3610    ///
3611    /// Blocking: this reads the journal files. Async callers use
3612    /// [`Self::read_durable_terminal_history`].
3613    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3614        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3615    }
3616
3617    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3618    /// read (up to every retained generation) never occupies a runtime worker.
3619    /// Fails only if the blocking task could not finish (runtime shutdown or a
3620    /// panic in the read).
3621    pub(crate) async fn read_durable_terminal_history(
3622        &self,
3623    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3624        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3625        let module_id = self.inner.module_id.clone();
3626        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3627            .await
3628    }
3629
3630    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3631        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3632            .map(|(status, _)| status)
3633    }
3634
3635    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3636        let state = lock_snapshot(&self.inner.snapshot)?;
3637        Ok(state
3638            .failed_retry
3639            .pending
3640            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3641            .map(|retry| {
3642                unix_ms_now().saturating_add(
3643                    u64::try_from(
3644                        retry
3645                            .deadline
3646                            .saturating_duration_since(Instant::now())
3647                            .as_millis(),
3648                    )
3649                    .unwrap_or(u64::MAX),
3650                )
3651            }))
3652    }
3653
3654    pub(crate) fn record_deliberate_severance(
3655        &self,
3656        identity: ProcessIdentity,
3657    ) -> Result<bool, SuperviseError> {
3658        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3659        if snapshot.pid != Some(identity.pid)
3660            || snapshot.process_start_time != Some(identity.start_time)
3661        {
3662            return Ok(false);
3663        }
3664        snapshot.deliberate_severance = Some(identity);
3665        Ok(true)
3666    }
3667
3668    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3669    ///
3670    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3671    /// does not produce reader-observability logs.
3672    pub(crate) fn status_for_control(
3673        &self,
3674        caller: &'static str,
3675    ) -> Result<ModuleStatus, SuperviseError> {
3676        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3677            .map(|(status, _)| status)
3678    }
3679
3680    fn status_with_snapshot_lock(
3681        &self,
3682        snapshot: &SharedSnapshot,
3683        caller: Option<&'static str>,
3684    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3685        let mut guard = match caller {
3686            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3687            None => lock_snapshot(snapshot)?,
3688        };
3689        // Read the budget through the pruning path so a reader sees the same
3690        // in-window count the restart decision would use, not a stale total.
3691        let restart_count =
3692            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3693        let snapshot = guard.clone();
3694        drop(guard);
3695        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3696            SuperviseError::StatePoisoned {
3697                module_id: Some(self.inner.module_id.clone()),
3698            }
3699        })?;
3700        let registration_active = self
3701            .inner
3702            .registry
3703            .get_module(&self.inner.module_id)
3704            .map_err(SuperviseError::Registry)?
3705            .is_some();
3706        let protocol = snapshot
3707            .spawned_protocol
3708            .unwrap_or(self.declared_protocol()?);
3709        let running_process =
3710            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3711        // Registration is the difference between the two protocols and the only
3712        // one: a subc module that has not registered cannot serve a request even
3713        // though its process is up, and a `none` module never registers at all,
3714        // so requiring it there would pin `live` to false for the whole life of
3715        // a perfectly healthy process.
3716        let live = match protocol {
3717            ModuleProtocol::Subc => running_process && registration_active,
3718            ModuleProtocol::None => running_process,
3719        };
3720
3721        Ok((
3722            ModuleStatus {
3723                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3724                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3725                module_id: self.inner.module_id.clone(),
3726                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3727                state: snapshot.state,
3728                enabled: snapshot.enabled,
3729                process_alive: snapshot.process_alive,
3730                registration_active,
3731                protocol,
3732                live,
3733                restart_count,
3734                lifetime_restarts: snapshot.lifetime_restarts,
3735                spawn_generation: snapshot.spawn_generation,
3736                max_restarts: self.inner.restart_policy.max_restarts,
3737                restart_window: self.inner.restart_policy.window,
3738                drain_timeout,
3739                restart_backoff: self.inner.restart_policy.backoff,
3740                restart_max_backoff: self.inner.restart_policy.max_backoff,
3741                pid: snapshot.reported_pid(),
3742                spawned_at_ms: snapshot.spawned_at_ms,
3743                spawned_from: snapshot.spawned_from,
3744                launch_spec_digest: snapshot.launch_spec_digest,
3745                process_start_time: snapshot.process_start_time,
3746                last_exit: snapshot.last_exit,
3747                health: snapshot.health,
3748            },
3749            snapshot.spawned_file_identity.clone(),
3750        ))
3751    }
3752
3753    #[cfg(test)]
3754    pub(crate) fn hold_snapshot_for_test(
3755        &self,
3756        acquired: std::sync::mpsc::Sender<()>,
3757        hold: Duration,
3758    ) -> std::thread::JoinHandle<()> {
3759        let snapshot = Arc::clone(&self.inner.snapshot);
3760        std::thread::spawn(move || {
3761            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3762            acquired
3763                .send(())
3764                .expect("test receiver waits for snapshot lock");
3765            std::thread::sleep(hold);
3766        })
3767    }
3768
3769    /// The status and the running-image check for `supervisor.provenance`,
3770    /// taken from one status read. The exec acknowledgement can land between
3771    /// two separate reads, and the reply would then pair "no pid yet" with an
3772    /// image observed after the module started, which describes no single
3773    /// moment.
3774    pub(crate) async fn status_and_running_image_agreement(
3775        &self,
3776    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3777        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3778        let image = self
3779            .inner
3780            .provenance_probe
3781            .observe(
3782                status.pid,
3783                status.spawned_from.as_deref(),
3784                identity,
3785                status.process_start_time,
3786            )
3787            .await;
3788        Ok((status, image))
3789    }
3790
3791    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3792        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3793            Ok(snapshot) => snapshot.clone(),
3794            Err(_) => {
3795                return subc_control::RunningImageAgreement::Unavailable {
3796                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3797                };
3798            }
3799        };
3800        self.inner
3801            .provenance_probe
3802            .observe(
3803                snapshot.reported_pid(),
3804                snapshot.spawned_from.as_deref(),
3805                snapshot.spawned_file_identity.clone(),
3806                snapshot.process_start_time,
3807            )
3808            .await
3809    }
3810
3811    /// Memory and CPU time of the module's current process, read now. Only the
3812    /// process the supervisor spawned is read, not processes it has started.
3813    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3814        #[cfg(windows)]
3815        {
3816            let image = lock_snapshot(&self.inner.snapshot)
3817                .ok()
3818                .and_then(|snapshot| {
3819                    snapshot
3820                        .spawned_file_identity
3821                        .as_ref()
3822                        .and_then(|identity| identity.windows_image.clone())
3823                });
3824            match image {
3825                Some(image) => {
3826                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3827                }
3828                None => subc_control::ChildResourceUsage::Unavailable {
3829                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3830                },
3831            }
3832        }
3833        #[cfg(not(windows))]
3834        {
3835            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3836                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3837                Err(_) => {
3838                    return subc_control::ChildResourceUsage::Unavailable {
3839                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3840                    }
3841                }
3842            };
3843            crate::child_resources::read(pid, start_time)
3844        }
3845    }
3846
3847    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3848        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3849        Ok(match snapshot.state {
3850            ModuleState::Restarting => true,
3851            ModuleState::Failed | ModuleState::Disabled => false,
3852            _ if snapshot.enabled && snapshot.respawn_pending => true,
3853            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3854        })
3855    }
3856
3857    #[cfg(test)]
3858    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3859        self.is_warming_with_snapshot_lock(None)
3860    }
3861
3862    pub(crate) fn is_warming_for_control(
3863        &self,
3864        caller: &'static str,
3865    ) -> Result<bool, SuperviseError> {
3866        self.is_warming_with_snapshot_lock(Some(caller))
3867    }
3868
3869    fn is_warming_with_snapshot_lock(
3870        &self,
3871        caller: Option<&'static str>,
3872    ) -> Result<bool, SuperviseError> {
3873        let snapshot = match caller {
3874            Some(caller) => {
3875                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3876            }
3877            None => lock_snapshot(&self.inner.snapshot)?,
3878        }
3879        .clone();
3880        Ok(matches!(
3881            snapshot.state,
3882            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3883        ))
3884    }
3885
3886    /// Drain the module and stop monitoring it.
3887    pub async fn drain(&self) -> Result<(), SuperviseError> {
3888        self.stop().await
3889    }
3890
3891    pub(crate) async fn retire_requested(
3892        &self,
3893        requester: Option<Principal>,
3894    ) -> Result<(), SuperviseError> {
3895        match self.state()? {
3896            ModuleState::Stopped => return Ok(()),
3897            ModuleState::Failed
3898                if lock_snapshot(&self.inner.snapshot)?
3899                    .failed_retry
3900                    .pending
3901                    .is_none() =>
3902            {
3903                return Ok(())
3904            }
3905            ModuleState::Failed
3906            | ModuleState::Starting
3907            | ModuleState::Running
3908            | ModuleState::Unresponsive
3909            | ModuleState::Restarting
3910            | ModuleState::Draining
3911            | ModuleState::Disabled => {}
3912        }
3913
3914        let (reply_tx, reply_rx) = oneshot::channel();
3915        self.inner
3916            .commands
3917            .send(SupervisorCommand::Retire {
3918                requester,
3919                reply: reply_tx,
3920            })
3921            .await
3922            .map_err(|_| SuperviseError::CommandClosed {
3923                module_id: self.inner.module_id.clone(),
3924            })?;
3925        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926            module_id: self.inner.module_id.clone(),
3927        })?
3928    }
3929
3930    pub async fn stop(&self) -> Result<(), SuperviseError> {
3931        self.stop_requested(None).await
3932    }
3933
3934    pub(crate) async fn stop_requested(
3935        &self,
3936        requester: Option<Principal>,
3937    ) -> Result<(), SuperviseError> {
3938        match self.state()? {
3939            ModuleState::Stopped => return Ok(()),
3940            ModuleState::Failed
3941                if lock_snapshot(&self.inner.snapshot)?
3942                    .failed_retry
3943                    .pending
3944                    .is_none() =>
3945            {
3946                return Ok(())
3947            }
3948            ModuleState::Failed
3949            | ModuleState::Starting
3950            | ModuleState::Running
3951            | ModuleState::Unresponsive
3952            | ModuleState::Restarting
3953            | ModuleState::Draining
3954            | ModuleState::Disabled => {}
3955        }
3956
3957        let (reply_tx, reply_rx) = oneshot::channel();
3958        self.inner
3959            .commands
3960            .send(SupervisorCommand::Drain {
3961                requester,
3962                reply: reply_tx,
3963            })
3964            .await
3965            .map_err(|_| SuperviseError::CommandClosed {
3966                module_id: self.inner.module_id.clone(),
3967            })?;
3968        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3969            module_id: self.inner.module_id.clone(),
3970        })?
3971    }
3972
3973    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3974        self.restart_requested(drain_timeout_ms, None).await
3975    }
3976
3977    pub(crate) async fn restart_requested(
3978        &self,
3979        drain_timeout_ms: Option<u64>,
3980        requester: Option<Principal>,
3981    ) -> Result<(), SuperviseError> {
3982        let queued_at = Instant::now();
3983        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3984        let (reply_tx, reply_rx) = oneshot::channel();
3985        self.inner
3986            .commands
3987            .send(SupervisorCommand::Restart {
3988                requester,
3989                drain_timeout_ms,
3990                received_at_generation,
3991                queued_at,
3992                reply: reply_tx,
3993            })
3994            .await
3995            .map_err(|_| SuperviseError::CommandClosed {
3996                module_id: self.inner.module_id.clone(),
3997            })?;
3998        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3999            module_id: self.inner.module_id.clone(),
4000        })?
4001    }
4002
4003    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
4004    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4005    /// process then drains in the background of the supervise loop) or has
4006    /// failed, leaving the old process serving.
4007    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4008        self.swap_requested(ready_timeout, None).await
4009    }
4010
4011    pub(crate) async fn swap_requested(
4012        &self,
4013        ready_timeout: Option<Duration>,
4014        requester: Option<Principal>,
4015    ) -> Result<(), SuperviseError> {
4016        let (reply_tx, reply_rx) = oneshot::channel();
4017        self.inner
4018            .commands
4019            .send(SupervisorCommand::Swap {
4020                requester,
4021                ready_timeout,
4022                reply: reply_tx,
4023            })
4024            .await
4025            .map_err(|_| SuperviseError::CommandClosed {
4026                module_id: self.inner.module_id.clone(),
4027            })?;
4028        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4029            module_id: self.inner.module_id.clone(),
4030        })?
4031    }
4032
4033    pub async fn reload(&self) -> Result<(), SuperviseError> {
4034        self.reload_requested(None).await
4035    }
4036
4037    pub(crate) async fn reload_requested(
4038        &self,
4039        requester: Option<Principal>,
4040    ) -> Result<(), SuperviseError> {
4041        let (reply_tx, reply_rx) = oneshot::channel();
4042        self.inner
4043            .commands
4044            .send(SupervisorCommand::Reload {
4045                requester,
4046                reply: reply_tx,
4047            })
4048            .await
4049            .map_err(|_| SuperviseError::CommandClosed {
4050                module_id: self.inner.module_id.clone(),
4051            })?;
4052        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4053            module_id: self.inner.module_id.clone(),
4054        })?
4055    }
4056
4057    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4058        self.set_enabled_requested(enabled, None).await
4059    }
4060
4061    pub(crate) async fn set_enabled_requested(
4062        &self,
4063        enabled: bool,
4064        requester: Option<Principal>,
4065    ) -> Result<bool, SuperviseError> {
4066        let (reply_tx, reply_rx) = oneshot::channel();
4067        self.inner
4068            .commands
4069            .send(SupervisorCommand::SetEnabled {
4070                requester,
4071                enabled,
4072                reply: reply_tx,
4073            })
4074            .await
4075            .map_err(|_| SuperviseError::CommandClosed {
4076                module_id: self.inner.module_id.clone(),
4077            })?;
4078        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4079            module_id: self.inner.module_id.clone(),
4080        })?
4081    }
4082
4083    /// The current process's protocol, or the configured protocol when down.
4084    /// A rescan stores the next launch spec without changing how an existing
4085    /// process registers, serves routes, is probed, or exits.
4086    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4087        let configured = self
4088            .inner
4089            .configuration
4090            .lock()
4091            .map_err(|_| SuperviseError::StatePoisoned {
4092                module_id: Some(self.inner.module_id.clone()),
4093            })?
4094            .spec
4095            .protocol;
4096        let state = lock_snapshot(&self.inner.snapshot)?;
4097        Ok(state.spawned_protocol.unwrap_or(configured))
4098    }
4099
4100    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4101        let configuration =
4102            self.inner
4103                .configuration
4104                .lock()
4105                .map_err(|_| SuperviseError::StatePoisoned {
4106                    module_id: Some(self.inner.module_id.clone()),
4107                })?;
4108        Ok((configuration.spec.clone(), configuration.health.clone()))
4109    }
4110
4111    /// Replace this module's launch spec, keeping its health and drain policy,
4112    /// the way a rescan does for a changed config entry. The running process is
4113    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4114    #[cfg(any(test, feature = "test-support"))]
4115    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4116        let (_, health) = self.configuration()?;
4117        let drain_timeout_ms = u64::try_from(
4118            self.inner
4119                .effective_drain_timeout
4120                .lock()
4121                .unwrap_or_else(|poisoned| poisoned.into_inner())
4122                .as_millis(),
4123        )
4124        .ok();
4125        self.update_configuration(spec, health, drain_timeout_ms)
4126            .await
4127    }
4128
4129    pub(crate) async fn update_configuration(
4130        &self,
4131        spec: ModuleSpec,
4132        health: HealthConfig,
4133        drain_timeout_ms: Option<u64>,
4134    ) -> Result<(), SuperviseError> {
4135        if spec.module_id != self.inner.module_id {
4136            return Err(SuperviseError::InvalidSpec {
4137                reason: "a supervised module's module_id cannot be changed".to_string(),
4138            });
4139        }
4140        validate_spec(&spec)?;
4141        let (reply_tx, reply_rx) = oneshot::channel();
4142        self.inner
4143            .commands
4144            .send(SupervisorCommand::UpdateConfiguration {
4145                spec: spec.clone(),
4146                health: health.clone(),
4147                drain_timeout_ms,
4148                reply: reply_tx,
4149            })
4150            .await
4151            .map_err(|_| SuperviseError::CommandClosed {
4152                module_id: self.inner.module_id.clone(),
4153            })?;
4154        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4155            module_id: self.inner.module_id.clone(),
4156        })?;
4157        let mut configuration =
4158            self.inner
4159                .configuration
4160                .lock()
4161                .map_err(|_| SuperviseError::StatePoisoned {
4162                    module_id: Some(self.inner.module_id.clone()),
4163                })?;
4164        configuration.spec = spec;
4165        configuration.health = health;
4166        Ok(())
4167    }
4168}
4169
4170impl Drop for SupervisedModuleInner {
4171    fn drop(&mut self) {
4172        let Ok(mut monitor) = self.monitor.lock() else {
4173            return;
4174        };
4175        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4176            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4177                state.state = ModuleState::Stopped;
4178                clear_current_process_facts(state);
4179            });
4180            monitor.abort();
4181        }
4182        let _ = monitor.take();
4183    }
4184}
4185
4186#[derive(Debug)]
4187enum SupervisorCommand {
4188    Drain {
4189        requester: Option<Principal>,
4190        reply: oneshot::Sender<Result<(), SuperviseError>>,
4191    },
4192    Retire {
4193        requester: Option<Principal>,
4194        reply: oneshot::Sender<Result<(), SuperviseError>>,
4195    },
4196    Restart {
4197        requester: Option<Principal>,
4198        /// Operator override for this one restart's drain budget, in ms. `None`
4199        /// uses the module's configured/default budget; `Some(0)` cuts
4200        /// immediately (wedge bounce: a stuck request never settles, so
4201        /// waiting only delays recovery).
4202        drain_timeout_ms: Option<u64>,
4203        /// The module's `spawn_generation` when the request was received, before
4204        /// it waited in the command queue. A queued restart whose module has
4205        /// since spawned a newer process is already satisfied (see the handler).
4206        received_at_generation: u64,
4207        /// When the request entered the command queue, so the handler can log
4208        /// how long it waited behind the loop's other work.
4209        queued_at: Instant,
4210        reply: oneshot::Sender<Result<(), SuperviseError>>,
4211    },
4212    Reload {
4213        requester: Option<Principal>,
4214        reply: oneshot::Sender<Result<(), SuperviseError>>,
4215    },
4216    SetEnabled {
4217        requester: Option<Principal>,
4218        enabled: bool,
4219        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4220    },
4221    UpdateConfiguration {
4222        spec: ModuleSpec,
4223        health: HealthConfig,
4224        /// Per-module drain override from the new config; `None` re-resolves to
4225        /// the supervisor-wide default.
4226        drain_timeout_ms: Option<u64>,
4227        reply: oneshot::Sender<()>,
4228    },
4229    Swap {
4230        requester: Option<Principal>,
4231        /// How long the candidate may take to register and declare itself
4232        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4233        ready_timeout: Option<Duration>,
4234        /// Answered at cutover or failure; the incumbent's drain follows.
4235        reply: oneshot::Sender<Result<(), SuperviseError>>,
4236    },
4237}
4238
4239#[derive(Debug)]
4240pub enum SuperviseError {
4241    InvalidSpec {
4242        reason: String,
4243    },
4244    Spawn {
4245        program: PathBuf,
4246        source: io::Error,
4247        cgroup_path: Option<PathBuf>,
4248    },
4249    Cgroup {
4250        module_id: String,
4251        source: io::Error,
4252    },
4253    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4254    /// than spawn a reserved module without its identity binding.
4255    LaunchNonce {
4256        reason: String,
4257    },
4258    Wait {
4259        module_id: String,
4260        source: io::Error,
4261    },
4262    Kill {
4263        module_id: String,
4264        source: io::Error,
4265    },
4266    Forwarding(ForwardingError),
4267    Registry(RegistryError),
4268    ReloadUnavailable {
4269        module_id: String,
4270        reason: String,
4271    },
4272    /// An operator restart/reload was requested for a module that is currently
4273    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4274    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4275    /// by a restart, so these commands are rejected instead of re-enabling it.
4276    Disabled {
4277        module_id: String,
4278    },
4279    ReloadFailed {
4280        module_id: String,
4281        reason: String,
4282    },
4283    RegistrationStillActive {
4284        module_id: String,
4285        waited: Duration,
4286    },
4287    StatePoisoned {
4288        module_id: Option<String>,
4289    },
4290    CommandClosed {
4291        module_id: String,
4292    },
4293    /// A restart or reload arrived while a swap's candidate was warming. The
4294    /// swap owns the module until it cuts over or fails; a stop or disable
4295    /// would have aborted it instead.
4296    SwapInProgress {
4297        module_id: String,
4298    },
4299    /// A swap was refused before anything was spawned.
4300    SwapRefused {
4301        module_id: String,
4302        reason: SwapRefusal,
4303    },
4304    /// A swap spawned a candidate and gave up on it. The candidate has been
4305    /// killed and its slot freed; the incumbent was left serving and was never
4306    /// drained, except in the one `CutoverLost` case described on that arm.
4307    SwapFailed {
4308        module_id: String,
4309        arm: SwapFailureArm,
4310        detail: String,
4311        /// How the candidate exited, when it exited on its own before the
4312        /// supervisor gave up on it.
4313        candidate_exit: Option<ExitReport>,
4314    },
4315}
4316
4317/// Why a swap was refused before a candidate was spawned.
4318#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4319pub enum SwapRefusal {
4320    /// The module's config does not declare `overlap: "safe"`.
4321    OverlapExclusive,
4322    /// The module is not registered, so there is no incumbent to keep serving
4323    /// and nothing a swap would improve on; a plain restart is the tool.
4324    NotRegistered,
4325    /// The module does not speak the subc wire, so a candidate could never
4326    /// register or declare itself ready.
4327    ProtocolNone,
4328    /// The supervisor lacks the forwarding table (to cut routes over) or the
4329    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4330    NotConfigured,
4331    /// A swap is already open for this module.
4332    AlreadySwapping,
4333}
4334
4335impl SwapRefusal {
4336    pub fn as_str(self) -> &'static str {
4337        match self {
4338            Self::OverlapExclusive => "overlap_exclusive",
4339            Self::NotRegistered => "not_registered",
4340            Self::ProtocolNone => "protocol_none",
4341            Self::NotConfigured => "not_configured",
4342            Self::AlreadySwapping => "already_swapping",
4343        }
4344    }
4345}
4346
4347/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4348/// serving and undrained; see `CutoverLost`.
4349#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4350pub enum SwapFailureArm {
4351    /// The candidate process could not be started.
4352    SpawnFailed,
4353    /// The candidate did not register within the readiness budget.
4354    NeverRegistered,
4355    /// The candidate registered but did not declare itself ready in time.
4356    NeverReady,
4357    /// The candidate exited before cutover.
4358    CandidateExited,
4359    /// The candidate declared itself ready but failed its health probe.
4360    CandidateUnhealthy,
4361    /// An operator stop, disable or retire arrived while the candidate warmed.
4362    /// The candidate was killed and the operator's command then carried out on
4363    /// the incumbent.
4364    Interrupted,
4365    /// The candidate's connection closed at the moment of cutover. If it
4366    /// closed before forwarding moved, the incumbent is untouched. If it closed
4367    /// between the forwarding and registry halves of cutover, forwarding can no
4368    /// longer route to the incumbent, so the module is restarted plainly.
4369    CutoverLost,
4370}
4371
4372impl SwapFailureArm {
4373    pub fn as_str(self) -> &'static str {
4374        match self {
4375            Self::SpawnFailed => "spawn_failed",
4376            Self::NeverRegistered => "never_registered",
4377            Self::NeverReady => "never_ready",
4378            Self::CandidateExited => "candidate_exited",
4379            Self::CandidateUnhealthy => "candidate_unhealthy",
4380            Self::Interrupted => "interrupted",
4381            Self::CutoverLost => "cutover_lost",
4382        }
4383    }
4384}
4385
4386impl fmt::Display for SuperviseError {
4387    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4388        match self {
4389            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4390            Self::Spawn {
4391                program,
4392                source,
4393                cgroup_path: Some(cgroup_path),
4394            } => write!(
4395                f,
4396                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4397                cgroup_path.display(),
4398                program.display()
4399            ),
4400            Self::Spawn {
4401                program,
4402                source,
4403                cgroup_path: None,
4404            } => write!(
4405                f,
4406                "failed to spawn module '{}': {source}",
4407                program.display()
4408            ),
4409            Self::Cgroup { module_id, source } => {
4410                write!(
4411                    f,
4412                    "failed to prepare cgroup for module '{module_id}': {source}"
4413                )
4414            }
4415            Self::LaunchNonce { reason } => {
4416                write!(
4417                    f,
4418                    "failed to generate reserved-module launch nonce: {reason}"
4419                )
4420            }
4421            Self::Wait { module_id, source } => {
4422                write!(f, "failed to wait for module '{module_id}': {source}")
4423            }
4424            Self::Kill { module_id, source } => {
4425                write!(f, "failed to kill module '{module_id}': {source}")
4426            }
4427            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4428            Self::Registry(err) => write!(f, "registry error: {err}"),
4429            Self::ReloadUnavailable { module_id, reason } => {
4430                write!(f, "reload unavailable for module '{module_id}': {reason}")
4431            }
4432            Self::Disabled { module_id } => {
4433                write!(
4434                    f,
4435                    "module '{module_id}' is disabled; enable it before restart or reload"
4436                )
4437            }
4438            Self::ReloadFailed { module_id, reason } => {
4439                write!(f, "reload failed for module '{module_id}': {reason}")
4440            }
4441            Self::RegistrationStillActive { module_id, waited } => write!(
4442                f,
4443                "module '{module_id}' registration remained active after waiting {waited:?}"
4444            ),
4445            Self::StatePoisoned { module_id } => match module_id {
4446                Some(module_id) => {
4447                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4448                }
4449                None => write!(f, "supervisor state was poisoned"),
4450            },
4451            Self::CommandClosed { module_id } => {
4452                write!(
4453                    f,
4454                    "supervisor command channel for module '{module_id}' is closed"
4455                )
4456            }
4457            Self::SwapInProgress { module_id } => write!(
4458                f,
4459                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4460            ),
4461            Self::SwapRefused { module_id, reason } => match reason {
4462                SwapRefusal::OverlapExclusive => write!(
4463                    f,
4464                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4465                ),
4466                SwapRefusal::NotRegistered => write!(
4467                    f,
4468                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4469                ),
4470                SwapRefusal::ProtocolNone => write!(
4471                    f,
4472                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4473                ),
4474                SwapRefusal::NotConfigured => write!(
4475                    f,
4476                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4477                ),
4478                SwapRefusal::AlreadySwapping => {
4479                    write!(f, "module '{module_id}' is already being swapped")
4480                }
4481            },
4482            Self::SwapFailed {
4483                module_id,
4484                arm,
4485                detail,
4486                ..
4487            } => write!(
4488                f,
4489                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4490                arm.as_str()
4491            ),
4492        }
4493    }
4494}
4495
4496impl Error for SuperviseError {
4497    fn source(&self) -> Option<&(dyn Error + 'static)> {
4498        match self {
4499            Self::Spawn { source, .. }
4500            | Self::Cgroup { source, .. }
4501            | Self::Wait { source, .. }
4502            | Self::Kill { source, .. } => Some(source),
4503            Self::Forwarding(err) => Some(err),
4504            Self::Registry(err) => Some(err),
4505            Self::LaunchNonce { .. }
4506            | Self::InvalidSpec { .. }
4507            | Self::ReloadUnavailable { .. }
4508            | Self::Disabled { .. }
4509            | Self::ReloadFailed { .. }
4510            | Self::RegistrationStillActive { .. }
4511            | Self::StatePoisoned { .. }
4512            | Self::CommandClosed { .. }
4513            | Self::SwapInProgress { .. }
4514            | Self::SwapRefused { .. }
4515            | Self::SwapFailed { .. } => None,
4516        }
4517    }
4518}
4519
4520pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4521    if spec.module_id.trim().is_empty() {
4522        return Err(SuperviseError::InvalidSpec {
4523            reason: "module_id must not be empty".to_string(),
4524        });
4525    }
4526
4527    Ok(())
4528}
4529
4530#[derive(Debug, Default)]
4531struct HealthProbeRuntime {
4532    spawn_generation: Option<u64>,
4533    configured_health: Option<HealthConfig>,
4534    registered_connection: Option<crate::ConnectionId>,
4535    advertised: bool,
4536    next_probe_at: Option<Instant>,
4537    probe_index: u64,
4538}
4539
4540fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4541    lock_snapshot(snapshot)
4542        .ok()
4543        .and_then(|state| state.spawned_protocol)
4544        .unwrap_or(spec.protocol)
4545}
4546
4547impl HealthProbeRuntime {
4548    fn refresh_registration(
4549        &mut self,
4550        spec: &ModuleSpec,
4551        runtime: &SupervisorRuntimeConfig,
4552        registry: &Registry,
4553        snapshot: &SharedSnapshot,
4554    ) {
4555        let generation = lock_snapshot(snapshot)
4556            .ok()
4557            .map(|state| state.spawn_generation);
4558        if self.spawn_generation != generation {
4559            // A replacement cannot reuse a deadline that is already due for its
4560            // predecessor. HTTP children get a full cadence to start; wire
4561            // children still wait for their own health-advertising HELLO.
4562            *self = Self {
4563                spawn_generation: generation,
4564                ..Self::default()
4565            };
4566        }
4567        if self.configured_health.as_ref() != Some(&runtime.health) {
4568            self.configured_health = Some(runtime.health.clone());
4569            self.next_probe_at = None;
4570            self.registered_connection = None;
4571            self.probe_index = 0;
4572        }
4573        // A non-wire process never registers. Only an explicitly configured
4574        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4575        // health failure for that kind of process.
4576        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4577            self.registered_connection = None;
4578            self.advertised = runtime.health.http.is_some();
4579            if !self.advertised {
4580                self.next_probe_at = None;
4581                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4582                    let unknown = ModuleHealthStatus::default();
4583                    if state.health != unknown {
4584                        state.health = unknown;
4585                    }
4586                });
4587            } else if self.next_probe_at.is_none() {
4588                self.next_probe_at = Some(
4589                    Instant::now()
4590                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4591                );
4592            }
4593            return;
4594        }
4595
4596        let registration = match registry.get_module(&spec.module_id) {
4597            Ok(registration) => registration,
4598            Err(err) => {
4599                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4600                self.advertised = false;
4601                self.next_probe_at = None;
4602                return;
4603            }
4604        };
4605
4606        let Some(registration) = registration else {
4607            self.registered_connection = None;
4608            self.advertised = false;
4609            self.next_probe_at = None;
4610            return;
4611        };
4612
4613        let advertised = registration
4614            .control_ops
4615            .iter()
4616            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4617        if !advertised {
4618            self.registered_connection = Some(registration.connection_id);
4619            self.advertised = false;
4620            self.next_probe_at = None;
4621            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622                if state.health.status != SupervisorHealthStatus::Unknown
4623                    || state.health.consecutive_failures != 0
4624                    || state.health.last_probe_ms.is_some()
4625                    || state.health.detail.is_some()
4626                    || state.health.metrics.is_some()
4627                {
4628                    state.health.status = SupervisorHealthStatus::Unknown;
4629                    state.health.consecutive_failures = 0;
4630                    state.health.last_probe_ms = None;
4631                    state.health.detail = None;
4632                    state.health.metrics = None;
4633                }
4634            });
4635            return;
4636        }
4637
4638        let reregistered = self.registered_connection != Some(registration.connection_id);
4639        self.registered_connection = Some(registration.connection_id);
4640        self.advertised = true;
4641        if reregistered || self.next_probe_at.is_none() {
4642            self.probe_index = 0;
4643            self.next_probe_at = Some(
4644                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4645            );
4646            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4647                state.health.status = SupervisorHealthStatus::Unknown;
4648                state.health.consecutive_failures = 0;
4649                state.health.detail = None;
4650                state.health.metrics = None;
4651            });
4652        }
4653    }
4654
4655    fn wake_after(&self) -> Option<Duration> {
4656        if !self.advertised {
4657            return None;
4658        }
4659        self.next_probe_at
4660            .map(|next| next.saturating_duration_since(Instant::now()))
4661    }
4662
4663    fn due(&self) -> bool {
4664        self.advertised
4665            && self
4666                .next_probe_at
4667                .is_some_and(|next| Instant::now() >= next)
4668    }
4669
4670    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4671        self.probe_index = self.probe_index.wrapping_add(1);
4672        self.next_probe_at = Some(
4673            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4674        );
4675    }
4676}
4677
4678/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4679///
4680/// This was a struct with a single `message: String`, and every one of the
4681/// fifteen construction sites collapsed into it. Each site knows exactly what it
4682/// saw -- the lane is gone, the module did not answer in time, the module
4683/// answered with the wrong thing -- and `handle_health_probe_failure` then
4684/// treated all of them identically: increment a counter, compare to a threshold,
4685/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4686/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4687///
4688/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4689///
4690/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4691///   answer on it again.
4692/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4693///   AND with a perfectly healthy one that lost a CPU race -- which is what
4694///   happens under machine load, and is how this supervisor killed a healthy
4695///   module three times in one day.
4696/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4697///   Restarting on it is defensible, but it is not the silence case and should
4698///   never be counted as one.
4699/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4700///   anything, so it cannot be evidence about the module at all.
4701///
4702/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4703/// one that fires most often, and while every variant collapsed into one string
4704/// it carried the same weight as the strongest.
4705///
4706/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4707/// DESIGN and a reader stopping at it gets the build backwards: the restart
4708/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4709/// probes still increment the failure streak and drive escalation at the
4710/// threshold (see `is_proof_of_death` below for why that is deliberate and
4711/// what gates the change). Absence of evidence restarts modules today.
4712#[derive(Debug)]
4713enum HealthProbeEvidence {
4714    /// The module's control lane is gone. Proof of death.
4715    LaneDead,
4716    /// No reply within the deadline. Proves nothing about the module's state.
4717    NoAnswer,
4718    /// The module replied, but not with a usable health report. Proves it is alive.
4719    BadAnswer,
4720    /// The daemon could not ask. Says nothing about the module.
4721    Misconfigured,
4722}
4723
4724#[derive(Debug)]
4725struct HealthProbeError {
4726    evidence: HealthProbeEvidence,
4727    message: String,
4728}
4729
4730impl HealthProbeError {
4731    fn lane_dead(message: impl Into<String>) -> Self {
4732        Self::with(HealthProbeEvidence::LaneDead, message)
4733    }
4734
4735    fn no_answer(message: impl Into<String>) -> Self {
4736        Self::with(HealthProbeEvidence::NoAnswer, message)
4737    }
4738
4739    fn bad_answer(message: impl Into<String>) -> Self {
4740        Self::with(HealthProbeEvidence::BadAnswer, message)
4741    }
4742
4743    fn misconfigured(message: impl Into<String>) -> Self {
4744        Self::with(HealthProbeEvidence::Misconfigured, message)
4745    }
4746
4747    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4748        Self {
4749            evidence,
4750            message: message.into(),
4751        }
4752    }
4753
4754    /// Whether this observation is proof the module cannot serve.
4755    ///
4756    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4757    /// variant that fires under CPU starvation, and treating it as proof is the
4758    /// defect this enum exists to make impossible to reintroduce silently.
4759    ///
4760    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4761    /// to restart also needs a bound for the case it excludes -- a genuinely
4762    /// wedged module, alive but never answering -- and that bound must come from
4763    /// the distribution of real late-answer latencies, which nothing measures
4764    /// yet. Landing the classification first makes the later change a one-line
4765    /// decision against evidence that already exists, rather than two unproven
4766    /// changes at once.
4767    #[allow(dead_code)]
4768    fn is_proof_of_death(&self) -> bool {
4769        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4770    }
4771
4772    /// Short stable label for logs and the health snapshot.
4773    ///
4774    /// An operator reading `ck health` currently cannot tell "the module is gone"
4775    /// from "the module did not answer in five seconds", because both render as
4776    /// prose in the same field. These labels are what make the two
4777    /// distinguishable at a glance, and they are what a later restart-policy
4778    /// change will be argued from.
4779    fn label(&self) -> &'static str {
4780        match self.evidence {
4781            HealthProbeEvidence::LaneDead => "lane-dead",
4782            HealthProbeEvidence::NoAnswer => "no-answer",
4783            HealthProbeEvidence::BadAnswer => "bad-answer",
4784            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4785        }
4786    }
4787}
4788
4789impl fmt::Display for HealthProbeError {
4790    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4791        f.write_str(&self.message)
4792    }
4793}
4794
4795async fn run_health_probe_cycle(
4796    spec: &ModuleSpec,
4797    runtime: &SupervisorRuntimeConfig,
4798    registry: &Registry,
4799    process_liveness: &SupervisorProcessLiveness,
4800    snapshot: &SharedSnapshot,
4801    child: &mut Option<SupervisedChild>,
4802) {
4803    let now_ms = unix_ms_now();
4804    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4805        .then_some(runtime.health.http.as_deref())
4806        .flatten();
4807    let result = match http {
4808        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4809        None => probe_module_health(&spec.module_id, runtime, None).await,
4810    };
4811    match result {
4812        Ok(report) => {
4813            handle_health_report(
4814                spec,
4815                runtime,
4816                registry,
4817                process_liveness,
4818                snapshot,
4819                child,
4820                report,
4821                now_ms,
4822            )
4823            .await;
4824        }
4825        Err(err) => {
4826            if http.is_some() {
4827                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4828                    state.health.status = SupervisorHealthStatus::Failing;
4829                });
4830            }
4831            handle_health_probe_failure(
4832                spec,
4833                runtime,
4834                registry,
4835                process_liveness,
4836                snapshot,
4837                child,
4838                err,
4839                now_ms,
4840            )
4841            .await;
4842        }
4843    }
4844}
4845
4846pub(crate) struct HttpProbeTarget<'a> {
4847    address: std::net::SocketAddr,
4848    localhost: bool,
4849    authority: &'a str,
4850    path: String,
4851}
4852
4853/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4854/// or TLS. A URL cannot turn a local health check into an outbound connection.
4855pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4856    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4857        return Err("must not contain whitespace, controls, or a fragment".into());
4858    }
4859    let rest = url
4860        .strip_prefix("http://")
4861        .ok_or("must use plain http://")?;
4862    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4863    let (authority, suffix) = rest.split_at(split);
4864    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4865        ("::1", rest)
4866    } else {
4867        let split = authority.find(':').unwrap_or(authority.len());
4868        authority.split_at(split)
4869    };
4870    let ip: std::net::IpAddr = match host {
4871        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4872        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4873        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4874    };
4875    let port = if port.is_empty() {
4876        80
4877    } else {
4878        port.strip_prefix(':')
4879            .and_then(|p| p.parse::<u16>().ok())
4880            .filter(|p| *p > 0)
4881            .ok_or("must have a valid nonzero TCP port")?
4882    };
4883    let path = if suffix.is_empty() {
4884        "/".into()
4885    } else if suffix.starts_with('?') {
4886        format!("/{suffix}")
4887    } else {
4888        suffix.into()
4889    };
4890    Ok(HttpProbeTarget {
4891        address: std::net::SocketAddr::new(ip, port),
4892        localhost: host == "localhost",
4893        authority,
4894        path,
4895    })
4896}
4897
4898async fn probe_http_health(
4899    url: &str,
4900    deadline: Duration,
4901) -> Result<HealthReport, HealthProbeError> {
4902    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4903    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4904    // Keep partial diagnostics outside the timed future so cancellation does
4905    // not discard a status line or body bytes already received.
4906    let mut response_status = String::new();
4907    let mut body = Vec::new();
4908    let probe = async {
4909        // Resolve localhost ourselves so a hosts-file override cannot turn
4910        // this into an outbound request, while IPv6-only local servers work.
4911        let connection = match tokio::net::TcpStream::connect(target.address).await {
4912            Err(_) if target.localhost => {
4913                tokio::net::TcpStream::connect((
4914                    std::net::Ipv6Addr::LOCALHOST,
4915                    target.address.port(),
4916                ))
4917                .await
4918            }
4919            result => result,
4920        };
4921        let mut stream = connection.map_err(|error| {
4922            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4923        })?;
4924        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4925            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4926        let mut reader = BufReader::new(stream);
4927        let mut budget = 16 * 1024;
4928        let status = http_line(&mut reader, &mut budget).await?;
4929        let mut words = status.split_ascii_whitespace();
4930        let version = words.next();
4931        let code = words
4932            .next()
4933            .filter(|word| word.len() == 3)
4934            .and_then(|word| word.parse::<u16>().ok());
4935        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4936            || !code.is_some_and(|code| (100..600).contains(&code))
4937        {
4938            return Err(HealthProbeError::bad_answer(format!(
4939                "invalid HTTP status: {status}"
4940            )));
4941        }
4942        let code = code.expect("validated status code");
4943        response_status = status.clone();
4944        let mut length = None;
4945        let mut chunked = false;
4946        loop {
4947            let line = http_line(&mut reader, &mut budget).await?;
4948            if line.is_empty() {
4949                break;
4950            }
4951            if let Some((name, value)) = line.split_once(':') {
4952                if name.eq_ignore_ascii_case("content-length") {
4953                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4954                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4955                    })?);
4956                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4957                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4958                }
4959            }
4960        }
4961        if chunked {
4962            while body.len() < 200 {
4963                let line = http_line(&mut reader, &mut budget).await?;
4964                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4965                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4966                if size == 0 {
4967                    break;
4968                }
4969                let count = size.min((200 - body.len()) as u64) as usize;
4970                let start = body.len();
4971                (&mut reader)
4972                    .take(count as u64)
4973                    .read_to_end(&mut body)
4974                    .await
4975                    .map_err(|error| {
4976                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4977                    })?;
4978                if body.len() - start != count {
4979                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4980                }
4981                if size > count as u64 || body.len() == 200 {
4982                    break;
4983                }
4984                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4985                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4986                }
4987            }
4988        } else {
4989            reader
4990                .take(length.unwrap_or(200).min(200))
4991                .read_to_end(&mut body)
4992                .await
4993                .map_err(|error| {
4994                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4995                })?;
4996        }
4997        if (200..300).contains(&code) {
4998            Ok(HealthReport::ok())
4999        } else {
5000            Err(HealthProbeError::bad_answer(
5001                "HTTP health endpoint returned non-2xx",
5002            ))
5003        }
5004    };
5005    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5006        Err(HealthProbeError::no_answer(format!(
5007            "HTTP probe timed out after {deadline:?}"
5008        )))
5009    });
5010    if let Err(error) = &mut result {
5011        if !response_status.is_empty() {
5012            error.message = format!(
5013                "{}; {response_status}: {}",
5014                error.message,
5015                String::from_utf8_lossy(&body)
5016            );
5017        }
5018    }
5019    result
5020}
5021
5022async fn http_line(
5023    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5024    remaining: &mut usize,
5025) -> Result<String, HealthProbeError> {
5026    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5027    let mut line = Vec::new();
5028    (&mut *reader)
5029        .take(*remaining as u64)
5030        .read_until(b'\n', &mut line)
5031        .await
5032        .map_err(|error| {
5033            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5034        })?;
5035    *remaining -= line.len();
5036    if !line.ends_with(b"\r\n") {
5037        return Err(HealthProbeError::bad_answer(
5038            "HTTP headers are incomplete or exceed 16 KiB",
5039        ));
5040    }
5041    line.truncate(line.len() - 2);
5042    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5043}
5044
5045async fn probe_module_health(
5046    module_id: &str,
5047    runtime: &SupervisorRuntimeConfig,
5048    drain_deadline: Option<Instant>,
5049) -> Result<HealthReport, HealthProbeError> {
5050    let Some(forwarding) = runtime.forwarding.as_ref() else {
5051        return Err(HealthProbeError::misconfigured(
5052            "supervisor was not configured with a forwarding table",
5053        ));
5054    };
5055    let probe_started_at = Instant::now();
5056    let mut deadline = probe_started_at + runtime.health.deadline;
5057    if let Some(drain_deadline) = drain_deadline {
5058        deadline = deadline.min(drain_deadline);
5059    }
5060    let pending = if drain_deadline.is_some() {
5061        forwarding.begin_drain_health_probe_rpc_for(
5062            module_id,
5063            MODULE_CONTROL_OP_HEALTH_CHECK,
5064            probe_started_at,
5065            deadline,
5066        )
5067    } else {
5068        forwarding.begin_health_probe_rpc_for(
5069            module_id,
5070            MODULE_CONTROL_OP_HEALTH_CHECK,
5071            probe_started_at,
5072            deadline,
5073        )
5074    }
5075    .map_err(|err| {
5076        // The endpoint is not registered, so there is no live control lane to
5077        // ask. That is the module being absent, not slow.
5078        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5079    })?;
5080    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5081}
5082
5083/// [`probe_module_health`] for one endpoint rather than the id's active one.
5084///
5085/// A swap probes two processes that no by-id lookup reaches: its candidate
5086/// before cutover, and its superseded incumbent (for busy gauges) while the
5087/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5088/// bounds the by-id drain probe.
5089async fn probe_endpoint_health(
5090    endpoint: crate::ModuleEndpointId,
5091    runtime: &SupervisorRuntimeConfig,
5092    deadline_cap: Option<Instant>,
5093) -> Result<HealthReport, HealthProbeError> {
5094    let Some(forwarding) = runtime.forwarding.as_ref() else {
5095        return Err(HealthProbeError::misconfigured(
5096            "supervisor was not configured with a forwarding table",
5097        ));
5098    };
5099    let probe_started_at = Instant::now();
5100    let mut deadline = probe_started_at + runtime.health.deadline;
5101    if let Some(cap) = deadline_cap {
5102        deadline = deadline.min(cap);
5103    }
5104    let pending = forwarding
5105        .begin_endpoint_health_probe_rpc_for(
5106            endpoint,
5107            MODULE_CONTROL_OP_HEALTH_CHECK,
5108            probe_started_at,
5109            deadline,
5110        )
5111        .map_err(|err| {
5112            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5113        })?;
5114    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5115}
5116
5117/// Send a begun health probe and classify its answer.
5118async fn await_health_probe(
5119    forwarding: &ForwardingTable,
5120    pending: PendingModuleControlRpc,
5121    deadline: Instant,
5122    probe_budget: Duration,
5123) -> Result<HealthReport, HealthProbeError> {
5124    let PendingModuleControlRpc {
5125        endpoint,
5126        module_sink,
5127        negotiated_ver,
5128        corr,
5129        receiver,
5130    } = pending;
5131    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5132        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5133    })?;
5134    let frame = Frame::build_with_version(
5135        negotiated_ver,
5136        FrameType::Request,
5137        control_flags(),
5138        0,
5139        0,
5140        corr,
5141        body,
5142    )
5143    .map_err(|err| {
5144        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5145    })?;
5146
5147    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5148    // blocks waiting for capacity when the module's egress queue is full, and an
5149    // unbounded await here freezes the whole supervision actor (it stops polling
5150    // Child::wait and supervisor commands), making the module unrecoverable
5151    // in-band. On timeout the probe fails like any transport failure.
5152    match timeout_at(deadline, module_sink.send(frame)).await {
5153        Ok(Ok(())) => {}
5154        Ok(Err(err)) => {
5155            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5156            // A closed sink means the module's egress channel is gone -- the
5157            // receiving half is dropped when its connection tears down. Proof.
5158            return Err(HealthProbeError::lane_dead(format!(
5159                "failed to send health.check: {err}"
5160            )));
5161        }
5162        Err(_elapsed) => {
5163            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5164            // A full egress queue means the module is not draining its socket, which
5165            // is consistent with a wedged module AND with one whose reader is merely
5166            // starved. Silence, not proof.
5167            return Err(HealthProbeError::no_answer(
5168                "health.check send timed out before enqueue (module egress full)",
5169            ));
5170        }
5171    }
5172
5173    match timeout_at(deadline, receiver).await {
5174        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5175        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5176        // and those prove it is alive even though the probe failed.
5177        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5178            response.health_report().ok_or_else(|| {
5179                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5180            })
5181        }
5182        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5183            format!("health.check rejected: {}", body.message),
5184        )),
5185        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5186            Err(HealthProbeError::lane_dead(message))
5187        }
5188        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5189            Err(HealthProbeError::bad_answer(message))
5190        }
5191        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5192            Err(HealthProbeError::bad_answer(format!(
5193                "expected module-control op '{expected}', got '{actual}'"
5194            )))
5195        }
5196        // A reply that crosses the deadline before this waiter observes it is
5197        // still proof of life. The forwarding path records its end-to-end latency
5198        // before delivering this classification.
5199        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5200            "module answered health.check after its daemon deadline",
5201        )),
5202        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5203            "health.check waiter was canceled before the module responded",
5204        )),
5205        Err(_) => {
5206            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5207            Err(HealthProbeError::no_answer(format!(
5208                "module did not answer health.check within {probe_budget:?}"
5209            )))
5210        }
5211    }
5212}
5213
5214#[allow(clippy::too_many_arguments)]
5215async fn handle_health_report(
5216    spec: &ModuleSpec,
5217    runtime: &SupervisorRuntimeConfig,
5218    registry: &Registry,
5219    process_liveness: &SupervisorProcessLiveness,
5220    snapshot: &SharedSnapshot,
5221    child: &mut Option<SupervisedChild>,
5222    report: HealthReport,
5223    now_ms: u64,
5224) {
5225    let status = supervisor_health_status(report.status);
5226    let detail = report.detail.clone();
5227    let metrics = truncate_health_metrics(report.metrics);
5228    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5229        state.health.status = status;
5230        state.health.last_probe_ms = Some(now_ms);
5231        state.health.detail = detail.clone();
5232        state.health.metrics = metrics.clone();
5233        state.health.consecutive_failures = 0;
5234        state.failed_retry.observe_health(
5235            report.status == HealthStatus::Ok,
5236            runtime.restart_policy.window,
5237        );
5238    });
5239
5240    let action = match report.status {
5241        HealthStatus::Ok => return,
5242        HealthStatus::Degraded => runtime.health.on_degraded,
5243        HealthStatus::Failing => runtime.health.on_failing,
5244    };
5245    apply_l3_health_action(
5246        spec,
5247        runtime,
5248        registry,
5249        process_liveness,
5250        snapshot,
5251        child,
5252        status,
5253        detail.as_deref(),
5254        action,
5255        now_ms,
5256    )
5257    .await;
5258}
5259
5260#[allow(clippy::too_many_arguments)]
5261async fn handle_health_probe_failure(
5262    spec: &ModuleSpec,
5263    runtime: &SupervisorRuntimeConfig,
5264    registry: &Registry,
5265    process_liveness: &SupervisorProcessLiveness,
5266    snapshot: &SharedSnapshot,
5267    child: &mut Option<SupervisedChild>,
5268    err: HealthProbeError,
5269    now_ms: u64,
5270) {
5271    let threshold = runtime.health.failure_threshold.max(1);
5272    let mut failures = 0;
5273    // Carry the evidence class into the operator-visible detail. Without it,
5274    // "module did not answer within 5s" and "the control lane is gone" are two
5275    // prose strings in the same field, and the reader has to know the codebase to
5276    // tell which one is proof of anything.
5277    let detail = format!("[{}] {err}", err.label());
5278    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5279        state
5280            .failed_retry
5281            .observe_health(false, runtime.restart_policy.window);
5282        // A failed wire probe invalidates the last report, even before the
5283        // restart threshold. HTTP probes already mark failures as Failing.
5284        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5285            state.health.status = SupervisorHealthStatus::Unknown;
5286        }
5287        state.health.last_probe_ms = Some(now_ms);
5288        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5289        state.health.detail = Some(detail.clone());
5290        state.health.metrics = None;
5291        failures = state.health.consecutive_failures;
5292    });
5293
5294    if failures < threshold {
5295        warn!(
5296            module_id = %spec.module_id,
5297            consecutive_failures = failures,
5298            threshold,
5299            evidence = err.label(),
5300            detail = %detail,
5301            "health.check probe failed"
5302        );
5303        return;
5304    }
5305
5306    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5307        state.state = ModuleState::Unresponsive;
5308        state.health.status = SupervisorHealthStatus::Unresponsive;
5309    });
5310    // The evidence class is logged at the kill site because this is the line an
5311    // operator reads after an unexplained restart. A streak of `no-answer` under
5312    // machine load is the known false-positive shape; a `lane-dead` is not.
5313    if runtime.health.critical {
5314        error!(
5315            module_id = %spec.module_id,
5316            status = "unresponsive",
5317            evidence = err.label(),
5318            detail = %detail,
5319            "critical module health alert"
5320        );
5321    } else {
5322        warn!(
5323            module_id = %spec.module_id,
5324            status = "unresponsive",
5325            evidence = err.label(),
5326            detail = %detail,
5327            "module health threshold breached"
5328        );
5329    }
5330    if let Err(err) = health_restart_child(
5331        spec,
5332        runtime,
5333        registry,
5334        process_liveness,
5335        snapshot,
5336        child,
5337        SupervisorHealthStatus::Unresponsive,
5338        Some(&detail),
5339        now_ms,
5340    )
5341    .await
5342    {
5343        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5344    }
5345}
5346
5347#[allow(clippy::too_many_arguments)]
5348async fn apply_l3_health_action(
5349    spec: &ModuleSpec,
5350    runtime: &SupervisorRuntimeConfig,
5351    registry: &Registry,
5352    process_liveness: &SupervisorProcessLiveness,
5353    snapshot: &SharedSnapshot,
5354    child: &mut Option<SupervisedChild>,
5355    status: SupervisorHealthStatus,
5356    detail: Option<&str>,
5357    action: HealthAction,
5358    now_ms: u64,
5359) {
5360    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5361    match action {
5362        HealthAction::Report => {
5363            info!(
5364                module_id = %spec.module_id,
5365                status = ?status,
5366                detail,
5367                "module reported non-ok health"
5368            );
5369        }
5370        HealthAction::Alert => {
5371            error!(
5372                module_id = %spec.module_id,
5373                status = ?status,
5374                detail,
5375                "module health alert"
5376            );
5377        }
5378        HealthAction::Restart => {
5379            if let Err(err) = health_restart_child(
5380                spec,
5381                runtime,
5382                registry,
5383                process_liveness,
5384                snapshot,
5385                child,
5386                status,
5387                detail,
5388                now_ms,
5389            )
5390            .await
5391            {
5392                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5393            }
5394        }
5395    }
5396}
5397
5398#[allow(clippy::too_many_arguments)]
5399async fn health_restart_child(
5400    spec: &ModuleSpec,
5401    runtime: &SupervisorRuntimeConfig,
5402    registry: &Registry,
5403    process_liveness: &SupervisorProcessLiveness,
5404    snapshot: &SharedSnapshot,
5405    child: &mut Option<SupervisedChild>,
5406    status: SupervisorHealthStatus,
5407    detail: Option<&str>,
5408    now_ms: u64,
5409) -> Result<(), SuperviseError> {
5410    let (enabled, schedule) = {
5411        let mut state = lock_snapshot(snapshot)?;
5412        let enabled = state.enabled;
5413        let schedule = if enabled {
5414            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5415        } else {
5416            None
5417        };
5418        (enabled, schedule)
5419    };
5420
5421    if !enabled {
5422        return Err(SuperviseError::Disabled {
5423            module_id: spec.module_id.clone(),
5424        });
5425    }
5426
5427    if schedule.is_none() {
5428        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5429        error!(
5430            module_id = %spec.module_id,
5431            status = ?status,
5432            detail,
5433            max_restarts = runtime.restart_policy.max_restarts,
5434            window_secs = runtime.restart_policy.window.as_secs(),
5435            reason = %runtime.restart_policy.budget_exhausted_detail(),
5436            "health restart budget exhausted; marking module failed"
5437        );
5438        let stop_notice = begin_forwarding_drain_if_configured(
5439            spec,
5440            runtime,
5441            registry,
5442            snapshot,
5443            Some(true),
5444            RouteCloseReason::Disable,
5445        )
5446        .await?;
5447        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5448            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5449        })?;
5450        let result = drain_optional_child(
5451            &spec.module_id,
5452            spec.protocol,
5453            stop_notice,
5454            registry,
5455            runtime.forwarding.as_deref(),
5456            snapshot,
5457            &runtime.terminal_ring,
5458            &runtime.spawn_events,
5459            child,
5460            runtime.drain_timeout,
5461            ModuleState::Failed,
5462            Some(true),
5463            &runtime.failure_context(true),
5464        )
5465        .await;
5466        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5467            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5468                state.failed_retry.schedule(&spec.module_id);
5469            }
5470        })?;
5471        result?;
5472        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5473        return Ok(());
5474    }
5475
5476    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5477    let mut restart_count = 0;
5478    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5479        restart_count = state.crash_restarts.len();
5480        state.state = ModuleState::Unresponsive;
5481        state.health.status = status;
5482        state.health.last_action = Some(HealthAction::Restart.to_string());
5483        state.health.last_action_ms = Some(now_ms);
5484    })?;
5485    warn!(
5486        module_id = %spec.module_id,
5487        status = ?status,
5488        detail,
5489        restart_count,
5490        restart_in_window = schedule.restart_in_window,
5491        delay_ms = schedule.delay.as_millis() as u64,
5492        "health-triggered module restart"
5493    );
5494
5495    let stop_notice = begin_forwarding_drain_if_configured(
5496        spec,
5497        runtime,
5498        registry,
5499        snapshot,
5500        Some(true),
5501        RouteCloseReason::Restart,
5502    )
5503    .await?;
5504    drain_optional_child(
5505        &spec.module_id,
5506        spec.protocol,
5507        stop_notice,
5508        registry,
5509        runtime.forwarding.as_deref(),
5510        snapshot,
5511        &runtime.terminal_ring,
5512        &runtime.spawn_events,
5513        child,
5514        runtime.drain_timeout,
5515        ModuleState::Restarting,
5516        Some(true),
5517        &runtime.failure_context(true),
5518    )
5519    .await?;
5520    schedule_respawn(
5521        runtime,
5522        snapshot,
5523        &spec.module_id,
5524        schedule.delay,
5525        RespawnKind::Spawn,
5526        true,
5527    )
5528}
5529
5530fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5531    if let Some(reply) = runtime
5532        .deferred_reload_reply
5533        .lock()
5534        .unwrap_or_else(|p| p.into_inner())
5535        .take()
5536    {
5537        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5538            module_id: module_id.to_string(),
5539            reason: reason.to_string(),
5540        }));
5541    }
5542}
5543
5544fn schedule_respawn(
5545    runtime: &SupervisorRuntimeConfig,
5546    snapshot: &SharedSnapshot,
5547    module_id: &str,
5548    delay: Duration,
5549    kind: RespawnKind,
5550    daemon_initiated: bool,
5551) -> Result<(), SuperviseError> {
5552    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5553    update_snapshot(snapshot, Some(module_id), |state| {
5554        state.respawn_pending = true
5555    })?;
5556    *runtime
5557        .scheduled_respawn
5558        .lock()
5559        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5560        deadline: Instant::now() + delay,
5561        kind,
5562        daemon_initiated,
5563    });
5564    Ok(())
5565}
5566
5567fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5568    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5569        state.health.last_action = Some(action);
5570        state.health.last_action_ms = Some(now_ms);
5571    });
5572}
5573
5574fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5575    match status {
5576        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5577        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5578        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5579    }
5580}
5581
5582/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5583/// returned to every `supervisor.list` and `supervisor.health` caller.
5584///
5585/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5586/// path: that request exists to return a module's complete metrics object, and
5587/// `ck health <module-id>` documents it as the way to see what the cached view
5588/// truncates. The asymmetry is the feature.
5589///
5590/// So a new caller must decide which side it is on rather than assume the cap is
5591/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5592/// the truncation that path exists to avoid.
5593fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5594    let metrics = metrics?;
5595    match serde_json::to_vec(&metrics) {
5596        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5597            "truncated": true,
5598            "original_bytes": encoded.len(),
5599        })),
5600        Ok(_) | Err(_) => Some(metrics),
5601    }
5602}
5603
5604/// Spread health probes so a fleet-wide restart does not converge them.
5605///
5606/// The delay is derived from the module id and probe index rather than a random
5607/// source, so it is deterministic per module: a module keeps its own offset
5608/// across daemon restarts instead of re-rolling into a collision.
5609fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5610    if cadence.is_zero() {
5611        return Duration::ZERO;
5612    }
5613    let cadence_ms = cadence.as_millis() as u64;
5614    // This early return is REDUNDANT, deliberately, and a mutation run will show
5615    // it surviving removal. Recording why here so the next person to notice does
5616    // not have to re-derive it:
5617    //
5618    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5619    //   a zero cadence and builds the Duration from whole milliseconds, so a
5620    //   sub-millisecond cadence cannot come from config.
5621    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5622    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5623    //   -- exactly what this returns.
5624    //
5625    // Kept as a guard against a future widening of the config parser (accepting
5626    // microseconds, say), which would make the sub-millisecond case reachable.
5627    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5628    // divides by zero. Remove this and nothing changes.
5629    if cadence_ms == 0 {
5630        return cadence;
5631    }
5632    // Note that this never returns less than one cadence, including for the FIRST
5633    // probe. So a freshly registered module reports health `unknown` for a full
5634    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5635    // ready to answer.
5636    //
5637    // That is a property of the supervisor's schedule, not of any module: an
5638    // operator watching a restart sees `unknown` and cannot tell it from a module
5639    // that is slow to warm. Measured on two unrelated modules, both flipping to
5640    // `ok` between 22s and 32s after restart.
5641    //
5642    // Left as-is because spreading the first probe is what keeps a fleet-wide
5643    // restart from firing fourteen simultaneous probes into a cold machine. The
5644    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5645    // that thundering herd for a faster first reading.
5646    let jitter_span = (cadence_ms / 10).max(1);
5647    let hash = module_id.as_bytes().iter().fold(
5648        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5649        |acc, byte| {
5650            acc.wrapping_mul(1099511628211)
5651                .wrapping_add(u64::from(*byte))
5652        },
5653    );
5654    cadence + Duration::from_millis(hash % jitter_span)
5655}
5656
5657#[cfg(test)]
5658mod tests {
5659    use super::*;
5660
5661    #[test]
5662    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5663        let handle = SupervisorHandle::new();
5664        let module_id = "readded-tombstone";
5665        handle.record_rescan_removal(module_id);
5666        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5667
5668        handle.apply_identity_configuration(&ModuleSpec {
5669            module_id: module_id.to_string(),
5670            program: PathBuf::from("/test/module"),
5671            args: Vec::new(),
5672            env: Vec::new(),
5673            reserved: false,
5674            reserved_prefixes: Vec::new(),
5675            protocol: ModuleProtocol::Subc,
5676            overlap: Default::default(),
5677        });
5678
5679        assert!(
5680            handle.removal_tombstone_age_ms(module_id).is_none(),
5681            "a re-added module must not retain a stale removal tombstone"
5682        );
5683    }
5684
5685    /// What one module's owner looked like from the control plane at the
5686    /// instant after its first process was spawned.
5687    #[derive(Debug, PartialEq, Eq)]
5688    struct OwnerInSpawnWindow {
5689        module_id: String,
5690        configured: bool,
5691        on_roster: bool,
5692        admission_refusal: Option<&'static str>,
5693    }
5694
5695    /// A supervised module's process can connect, register, sync its scopes
5696    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5697    /// the module on the roster until afterward; configured supervision does
5698    /// so before spawning. In both cases the owner must
5699    /// already read as configured, so a scoped `route.open` against it is
5700    /// refused as retryable `scope_not_synced` and not as terminal
5701    /// `scope_not_live` ("will never sync").
5702    ///
5703    /// The hook runs just after each initial spawn attempt, before returning to
5704    /// the caller, so no race with a real child's registration is needed.
5705    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5706    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5707        use crate::scopes::ScopeTable;
5708        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5709
5710        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5711        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5712            module_id: module_id.to_string(),
5713            program,
5714            args: Vec::new(),
5715            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5716                .into_iter()
5717                .map(|key| (key.to_string(), dir.path().display().to_string()))
5718                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5719                .collect(),
5720            reserved: false,
5721            reserved_prefixes: Vec::new(),
5722            protocol: ModuleProtocol::Subc,
5723            overlap: Default::default(),
5724        };
5725        let live = super::terminal_history_tests::fake_aft_stub_path();
5726        let missing = dir.path().join("definitely-missing-module");
5727
5728        let handle = SupervisorHandle::new();
5729        let mut supervisor =
5730            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5731                .with_handle(handle.clone());
5732        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5733        let hook_handle = handle.clone();
5734        let hook_observed = Arc::clone(&observed);
5735        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5736            // Exactly what the control plane computes for a scoped route.open
5737            // naming this module as the owner of a scope it has not synced.
5738            let configured = hook_handle.is_configured(module_id);
5739            let selector = ScopeSelector::new(
5740                Principal::Reserved {
5741                    module_id: module_id.to_string(),
5742                },
5743                "s".to_string(),
5744                Some(1),
5745            );
5746            let carrier = Principal::Reserved {
5747                module_id: "carrier".to_string(),
5748            };
5749            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5750                .admit(&carrier, module_id, &selector, configured)
5751            {
5752                Ok(_) => None,
5753                Err(refusal) => Some(refusal.code),
5754            };
5755            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5756                module_id: module_id.to_string(),
5757                configured,
5758                on_roster: hook_handle.get(module_id).is_some(),
5759                admission_refusal,
5760            });
5761        })));
5762
5763        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5764        let configured = supervisor
5765            .supervise_configured(stub("configured", live.clone()), true)
5766            .unwrap();
5767        let with_health = supervisor
5768            .supervise_configured_with_health(
5769                stub("with-health", live.clone()),
5770                true,
5771                HealthConfig::default(),
5772                None,
5773                RestartPolicy::default(),
5774            )
5775            .unwrap();
5776        // The failed-spawn path still puts the module on the roster (as
5777        // failed), so it is configured throughout.
5778        let failed = supervisor
5779            .supervise_configured_with_health(
5780                stub("failed-spawn", missing.clone()),
5781                true,
5782                HealthConfig::default(),
5783                None,
5784                RestartPolicy::default(),
5785            )
5786            .unwrap();
5787        // A failed plain `spawn` puts nothing on the roster, so its mark is
5788        // taken back once the spawn has failed.
5789        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5790
5791        let expected = [
5792            "plain",
5793            "configured",
5794            "with-health",
5795            "failed-spawn",
5796            "spawn-error",
5797        ]
5798        .into_iter()
5799        .map(|module_id| OwnerInSpawnWindow {
5800            module_id: module_id.to_string(),
5801            configured: true,
5802            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5803            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5804        })
5805        .collect::<Vec<_>>();
5806        assert_eq!(*observed.lock().unwrap(), expected);
5807
5808        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5809            assert!(
5810                handle.get(module_id).is_some(),
5811                "{module_id} is on the roster"
5812            );
5813            assert!(
5814                handle.is_configured(module_id),
5815                "{module_id} stays configured"
5816            );
5817        }
5818        assert!(handle.get("spawn-error").is_none());
5819        assert!(
5820            !handle.is_configured("spawn-error"),
5821            "a plain spawn that failed must not leave its module marked configured"
5822        );
5823
5824        // Leaving the roster clears the mark with it.
5825        handle.retire("failed-spawn");
5826        assert!(!handle.is_configured("failed-spawn"));
5827
5828        for module in [plain, configured, with_health] {
5829            module.stop().await.unwrap();
5830        }
5831        drop(failed);
5832    }
5833
5834    /// A configured module reads as starting until the supervisor has admitted
5835    /// its first process (exec confirmed, identity durably recorded), even
5836    /// though that process is already running by then and can register over
5837    /// HELLO, which is what lists it in catalog.list. So a client that needs
5838    /// the supervisor state must wait for that state; seeing the module in
5839    /// the catalog does not mean supervisor.list already says running.
5840    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5841    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5842        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5843        let spec = ModuleSpec {
5844            module_id: "spawned-not-admitted".to_string(),
5845            program: super::terminal_history_tests::fake_aft_stub_path(),
5846            args: Vec::new(),
5847            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5848                .into_iter()
5849                .map(|key| (key.to_string(), dir.path().display().to_string()))
5850                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5851                .collect(),
5852            reserved: false,
5853            reserved_prefixes: Vec::new(),
5854            protocol: ModuleProtocol::Subc,
5855            overlap: Default::default(),
5856        };
5857        let handle = SupervisorHandle::new();
5858        let mut supervisor =
5859            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5860                .with_handle(handle.clone());
5861        let observed = Arc::new(Mutex::new(None));
5862        let hook_handle = handle.clone();
5863        let hook_observed = Arc::clone(&observed);
5864        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5865            let state = hook_handle
5866                .get(module_id)
5867                .expect("a configured module is on the roster before it is spawned")
5868                .state()
5869                .unwrap();
5870            *hook_observed.lock().unwrap() = Some(state);
5871        })));
5872
5873        let module = supervisor.supervise_configured(spec, true).unwrap();
5874        assert_eq!(
5875            *observed.lock().unwrap(),
5876            Some(ModuleState::Starting),
5877            "between the spawn and its admission the supervisor still reports starting"
5878        );
5879        assert_eq!(module.state().unwrap(), ModuleState::Running);
5880        module.stop().await.unwrap();
5881    }
5882
5883    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5884        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5885        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5886            snapshot.process_alive = true;
5887            snapshot.pid = Some(41);
5888            snapshot.spawned_at_ms = Some(42);
5889            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5890            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5891            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5892                device: 43,
5893                inode: 44,
5894                #[cfg(windows)]
5895                windows_image: None,
5896            });
5897        })
5898        .unwrap();
5899        snapshot
5900    }
5901
5902    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5903        let snapshot = lock_snapshot(snapshot).unwrap();
5904        assert!(!snapshot.process_alive);
5905        assert_eq!(snapshot.pid, None);
5906        assert_eq!(snapshot.spawned_at_ms, None);
5907        assert_eq!(snapshot.spawned_from, None);
5908        assert_eq!(snapshot.launch_spec_digest, None);
5909        assert_eq!(snapshot.spawned_file_identity, None);
5910    }
5911
5912    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5913    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5914        let supervisor =
5915            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5916        let mut runtime = supervisor.runtime_config();
5917        runtime.test_seed_stale_facts_before_enable_spawn = true;
5918        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5919        let mut child = None;
5920        let spec = ModuleSpec {
5921            module_id: "failed-enable-clears-facts".to_string(),
5922            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5923            args: Vec::new(),
5924            env: Vec::new(),
5925            reserved: false,
5926            reserved_prefixes: Vec::new(),
5927            protocol: ModuleProtocol::Subc,
5928            overlap: Default::default(),
5929        };
5930
5931        let result = set_child_enabled(
5932            &spec,
5933            &runtime,
5934            &supervisor.registry,
5935            &supervisor.process_liveness,
5936            &snapshot,
5937            &mut child,
5938            true,
5939            &runtime.failure_context(false),
5940        )
5941        .await;
5942
5943        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5944        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5945        assert_snapshot_process_facts_cleared(&snapshot);
5946    }
5947
5948    #[tokio::test]
5949    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5950        let supervisor =
5951            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5952        let runtime = supervisor.runtime_config();
5953        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5954            ModuleState::Restarting,
5955            true,
5956        )));
5957        let spec = ModuleSpec {
5958            module_id: "start-stranded-restarting".to_string(),
5959            program: super::terminal_history_tests::fake_aft_stub_path(),
5960            args: Vec::new(),
5961            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5962            reserved: false,
5963            reserved_prefixes: Vec::new(),
5964            protocol: ModuleProtocol::None,
5965            overlap: Default::default(),
5966        };
5967        let mut child = None;
5968        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5969        assert!(!super::set_child_enabled(
5970            &spec,
5971            &runtime,
5972            &Registry::default(),
5973            &supervisor.process_liveness,
5974            &snapshot,
5975            &mut child,
5976            true,
5977            &runtime.failure_context(false),
5978        )
5979        .await
5980        .unwrap());
5981        assert!(child.is_none());
5982        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5983        assert!(super::set_child_enabled(
5984            &spec,
5985            &runtime,
5986            &Registry::default(),
5987            &supervisor.process_liveness,
5988            &snapshot,
5989            &mut child,
5990            true,
5991            &runtime.failure_context(false),
5992        )
5993        .await
5994        .unwrap());
5995        assert_eq!(
5996            lock_snapshot(&snapshot).unwrap().state,
5997            ModuleState::Running
5998        );
5999        let mut child = child.unwrap();
6000        child.start_kill().unwrap();
6001        child.wait().await.unwrap();
6002    }
6003
6004    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6005    async fn failed_reload_spawn_clears_current_process_facts() {
6006        let supervisor =
6007            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6008        let mut runtime = supervisor.runtime_config();
6009        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6010        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6011        let mut child = None;
6012        let spec = ModuleSpec {
6013            module_id: "failed-reload-clears-facts".to_string(),
6014            program: PathBuf::from("/unused/failed-reload-module"),
6015            args: Vec::new(),
6016            env: Vec::new(),
6017            reserved: false,
6018            reserved_prefixes: Vec::new(),
6019            protocol: ModuleProtocol::Subc,
6020            overlap: Default::default(),
6021        };
6022
6023        let result = handle_reload_spawn_failure(
6024            &spec,
6025            &runtime,
6026            &supervisor.process_liveness,
6027            &snapshot,
6028            &mut child,
6029            "forced reload spawn failure".to_string(),
6030        )
6031        .await;
6032
6033        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6034        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6035        assert_snapshot_process_facts_cleared(&snapshot);
6036    }
6037
6038    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6039    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6040        let supervisor =
6041            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6042        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6043        let module = supervisor.supervised_module(
6044            ModuleSpec {
6045                module_id: "drop-clears-facts".to_string(),
6046                program: PathBuf::from("/unused/drop-module"),
6047                args: Vec::new(),
6048                env: Vec::new(),
6049                reserved: false,
6050                reserved_prefixes: Vec::new(),
6051                protocol: ModuleProtocol::Subc,
6052                overlap: Default::default(),
6053            },
6054            supervisor.runtime_config(),
6055            Arc::clone(&snapshot),
6056            None,
6057        );
6058        assert!(!module
6059            .inner
6060            .monitor
6061            .lock()
6062            .unwrap()
6063            .as_ref()
6064            .unwrap()
6065            .is_finished());
6066
6067        drop(module);
6068
6069        assert_eq!(
6070            lock_snapshot(&snapshot).unwrap().state,
6071            ModuleState::Stopped
6072        );
6073        assert_snapshot_process_facts_cleared(&snapshot);
6074    }
6075
6076    #[cfg(unix)]
6077    #[tokio::test]
6078    async fn rescan_preserves_running_protocol_until_respawn() {
6079        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6080        let initial = ModuleSpec {
6081            module_id: "rescan-protocol".into(),
6082            program: PathBuf::from("/bin/sleep"),
6083            args: vec!["60".into()],
6084            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6085                .into_iter()
6086                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6087                .collect(),
6088            reserved: false,
6089            reserved_prefixes: vec![],
6090            protocol: ModuleProtocol::None,
6091            overlap: Default::default(),
6092        };
6093        let supervisor =
6094            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6095        let module = supervisor.spawn(initial.clone()).unwrap();
6096        assert!(module.status().unwrap().live);
6097        let mut next = initial;
6098        next.protocol = ModuleProtocol::Subc;
6099        module
6100            .update_configuration(next.clone(), HealthConfig::default(), None)
6101            .await
6102            .unwrap();
6103        assert!(
6104            module.status().unwrap().live,
6105            "rescan must not require HELLO from the old non-wire process"
6106        );
6107        let runtime = supervisor.runtime_config();
6108        let action = on_child_exit(
6109            &next,
6110            RestartPolicy::default(),
6111            &supervisor.registry,
6112            &module.inner.snapshot,
6113            &runtime.terminal_ring,
6114            &runtime.spawn_events,
6115            &runtime.child_roster,
6116            ExitReport {
6117                kind: ExitKind::Clean,
6118                code: Some(0),
6119                signal: None,
6120                at_ms: unix_ms_now(),
6121            },
6122            true,
6123        )
6124        .await;
6125        assert!(
6126            matches!(action, NextAction::Restart { .. }),
6127            "the old non-wire process's clean exit must restart"
6128        );
6129        module.drain().await.unwrap();
6130    }
6131
6132    #[cfg(unix)]
6133    #[tokio::test]
6134    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6135        use std::os::unix::fs::PermissionsExt;
6136        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6137        let script = dir.join("module.sh");
6138        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6139        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6140        let record_path = dir.join("live-children.json");
6141        let supervisor =
6142            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6143                .with_live_children_record(&record_path);
6144        for (program, args) in [
6145            (PathBuf::from("sleep"), vec!["60".into()]),
6146            (script, vec![]),
6147        ] {
6148            let spec = ModuleSpec {
6149                module_id: "image-identity".into(),
6150                program: program.clone(),
6151                args,
6152                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6153                    .into_iter()
6154                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6155                    .collect(),
6156                reserved: false,
6157                reserved_prefixes: vec![],
6158                protocol: ModuleProtocol::None,
6159                overlap: Default::default(),
6160            };
6161            let module = supervisor.spawn(spec).unwrap();
6162            #[cfg(target_os = "macos")]
6163            {
6164                // SETEXEC confirmation is asynchronous; the orphan record must
6165                // identify the final image, never the intermediate trampoline.
6166                let deadline = Instant::now() + Duration::from_secs(5);
6167                while crate::live_children::read_record(&record_path)
6168                    .unwrap()
6169                    .iter()
6170                    .all(|entry| entry.executable.is_none())
6171                {
6172                    assert!(Instant::now() < deadline, "module image was not confirmed");
6173                    tokio::time::sleep(Duration::from_millis(5)).await;
6174                }
6175            }
6176            let entry = crate::live_children::read_record(&record_path)
6177                .unwrap()
6178                .pop()
6179                .unwrap();
6180            let observed = subc_os::Process::open(entry.pid)
6181                .unwrap()
6182                .unwrap()
6183                .observe()
6184                .unwrap();
6185            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6186            module.drain().await.unwrap();
6187            assert_eq!(
6188                verdict,
6189                crate::live_children::IdentityVerdict::Matches,
6190                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6191            );
6192        }
6193    }
6194
6195    #[cfg(unix)]
6196    fn http_fixture(
6197        dir: &std::path::Path,
6198        url: &str,
6199        threshold: u32,
6200    ) -> crate::daemon_config::ConfiguredModule {
6201        let path = dir.join("subc.jsonc");
6202        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6203            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6204            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6205            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6206            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6207        }}}).to_string()).unwrap();
6208        crate::daemon_config::load(&path)
6209            .unwrap()
6210            .unwrap()
6211            .modules
6212            .pop()
6213            .unwrap()
6214    }
6215
6216    #[cfg(unix)]
6217    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6218        timeout(Duration::from_secs(5), async {
6219            loop {
6220                if module.status().unwrap().health.status == status {
6221                    break;
6222                }
6223                sleep(Duration::from_millis(5)).await;
6224            }
6225        })
6226        .await
6227        .unwrap_or_else(|_| {
6228            panic!(
6229                "expected {status:?}, got {:?}",
6230                module.status().unwrap().health
6231            )
6232        });
6233    }
6234
6235    #[cfg(unix)]
6236    #[tokio::test]
6237    async fn http_health_status_flips_ok_failing_ok() {
6238        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6239        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6240        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6241        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6242        let serving_status = status.clone();
6243        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6244        let server = tokio::spawn(async move {
6245            loop {
6246                let (mut stream, _) = listener.accept().await.unwrap();
6247                let mut request = [0u8; 2048];
6248                let count = stream.read(&mut request).await.unwrap();
6249                assert!(count > 0, "a probe must send an HTTP request");
6250                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6251                let body = if code == 200 {
6252                    "ready"
6253                } else {
6254                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6255                };
6256                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6257                let _ = stream.write_all(response.as_bytes()).await;
6258            }
6259        });
6260        let configured = http_fixture(&dir, &url, 1000);
6261        let module =
6262            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6263                .supervise_configured_with_health(
6264                    configured.module_spec(),
6265                    true,
6266                    configured.health,
6267                    configured.drain_timeout_ms,
6268                    configured.restart,
6269                )
6270                .unwrap();
6271        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6272        status.store(503, std::sync::atomic::Ordering::SeqCst);
6273        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6274        assert!(module
6275            .status()
6276            .unwrap()
6277            .health
6278            .detail
6279            .unwrap()
6280            .contains("scratch failure"));
6281        status.store(200, std::sync::atomic::Ordering::SeqCst);
6282        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6283        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6284        let before = module.status().unwrap();
6285        let (spec, mut health) = module.configuration().unwrap();
6286        health.http = None;
6287        module
6288            .update_configuration(spec.clone(), health.clone(), Some(10))
6289            .await
6290            .unwrap();
6291        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6292        health.http = Some(url);
6293        module
6294            .update_configuration(spec, health, Some(10))
6295            .await
6296            .unwrap();
6297        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6298        assert_eq!(
6299            module.status().unwrap().pid,
6300            before.pid,
6301            "changing a probe must apply live, not restart its process"
6302        );
6303        let (spec, mut health) = module.configuration().unwrap();
6304        health.failure_threshold = 2;
6305        module
6306            .update_configuration(spec, health, Some(10))
6307            .await
6308            .unwrap();
6309        status.store(503, std::sync::atomic::Ordering::SeqCst);
6310        timeout(Duration::from_secs(5), async {
6311            while module.status().unwrap().spawn_generation == before.spawn_generation {
6312                sleep(Duration::from_millis(5)).await;
6313            }
6314        })
6315        .await
6316        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6317        module.drain().await.unwrap();
6318        server.abort();
6319    }
6320
6321    #[cfg(unix)]
6322    #[tokio::test]
6323    async fn http_health_replacement_recovers_after_one_restart() {
6324        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6325        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6326        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6327        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6328        let mut configured = http_fixture(&dir, &url, 2);
6329        configured.restart.max_restarts = 2;
6330        let module =
6331            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6332                .supervise_configured_with_health(
6333                    configured.module_spec(),
6334                    true,
6335                    configured.health,
6336                    configured.drain_timeout_ms,
6337                    configured.restart,
6338                )
6339                .unwrap();
6340        let first_generation = module.status().unwrap().spawn_generation;
6341        let observed = module.clone();
6342        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6343        let server = tokio::spawn(async move {
6344            let mut generation = first_generation;
6345            let mut probes = 0;
6346            loop {
6347                let (mut stream, _) = listener.accept().await.unwrap();
6348                let mut request = [0; 2048];
6349                assert!(stream.read(&mut request).await.unwrap() > 0);
6350                let current = observed.status().unwrap().spawn_generation;
6351                if current != generation {
6352                    generation = current;
6353                    probes = 0;
6354                }
6355                probes += 1;
6356                // The original process never recovers. Each replacement has one
6357                // transient miss, then answers successfully for its remaining life.
6358                let healthy = generation != first_generation && probes > 1;
6359                let code = if healthy { 200 } else { 503 };
6360                let response = format!(
6361                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6362                );
6363                let _ = stream.write_all(response.as_bytes()).await;
6364                if healthy {
6365                    healthy_tx.send(()).await.unwrap();
6366                }
6367            }
6368        });
6369        timeout(Duration::from_secs(5), async {
6370            let mut healthy_probes = 0;
6371            loop {
6372                if module.state().unwrap() == ModuleState::Failed {
6373                    break;
6374                }
6375                while healthy_rx.try_recv().is_ok() {
6376                    healthy_probes += 1;
6377                }
6378                if healthy_probes >= 3
6379                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6380                {
6381                    break;
6382                }
6383                sleep(Duration::from_millis(5)).await;
6384            }
6385        })
6386        .await
6387        .expect("the replacement must settle healthy without exhausting its restart budget");
6388        let status = module.status().unwrap();
6389        module.drain().await.unwrap();
6390        server.abort();
6391        assert_eq!(status.state, ModuleState::Running);
6392        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6393        assert_eq!(status.health.consecutive_failures, 0);
6394        assert_eq!(status.lifetime_restarts, 1);
6395        assert_eq!(status.restart_count, 1);
6396    }
6397
6398    #[cfg(unix)]
6399    #[tokio::test]
6400    async fn http_health_sustained_failure_exhausts_restart_budget() {
6401        use std::collections::BTreeMap;
6402        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6403        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6404        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6405        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6406        let mut configured = http_fixture(&dir, &url, 2);
6407        configured.restart.max_restarts = 2;
6408        let module =
6409            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6410                .supervise_configured_with_health(
6411                    configured.module_spec(),
6412                    true,
6413                    configured.health,
6414                    configured.drain_timeout_ms,
6415                    configured.restart,
6416                )
6417                .unwrap();
6418        let observed = module.clone();
6419        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6420        let served = probes.clone();
6421        let server = tokio::spawn(async move {
6422            loop {
6423                let (mut stream, _) = listener.accept().await.unwrap();
6424                let mut request = [0; 2048];
6425                assert!(stream.read(&mut request).await.unwrap() > 0);
6426                *served
6427                    .lock()
6428                    .unwrap()
6429                    .entry(observed.status().unwrap().spawn_generation)
6430                    .or_default() += 1;
6431                let _ = stream
6432                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6433                    .await;
6434            }
6435        });
6436        timeout(Duration::from_secs(5), async {
6437            while module.state().unwrap() != ModuleState::Failed {
6438                sleep(Duration::from_millis(5)).await;
6439            }
6440        })
6441        .await
6442        .expect("sustained failures must still exhaust the health restart budget");
6443        let status = module.status().unwrap();
6444        server.abort();
6445        assert_eq!(status.restart_count, 2);
6446        assert_eq!(status.lifetime_restarts, 2);
6447        assert!(!status.process_alive);
6448        assert_eq!(
6449            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6450            vec![2, 2, 2]
6451        );
6452    }
6453
6454    #[cfg(unix)]
6455    #[tokio::test]
6456    async fn spawn_clears_the_previous_process_health() {
6457        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6458        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6459        configured.health.cadence = Duration::from_secs(3600);
6460        let module =
6461            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6462                .supervise_configured_with_health(
6463                    configured.module_spec(),
6464                    false,
6465                    configured.health,
6466                    configured.drain_timeout_ms,
6467                    configured.restart,
6468                )
6469                .unwrap();
6470        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6471            state.health.status = SupervisorHealthStatus::Unresponsive;
6472            state.health.consecutive_failures = 3;
6473            state.health.detail = Some("old process refused connections".into());
6474            state.health.metrics = Some(serde_json::json!({"old": true}));
6475            state.health.last_probe_ms = Some(123);
6476            state.health.last_action = Some("restart".into());
6477            state.health.last_action_ms = Some(124);
6478        })
6479        .unwrap();
6480        module.set_enabled(true).await.unwrap();
6481        let health = module.status().unwrap().health;
6482        module.drain().await.unwrap();
6483        assert_eq!(health.consecutive_failures, 0);
6484        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6485        assert_eq!(health.detail, None);
6486        assert_eq!(health.metrics, None);
6487        assert_eq!(health.last_probe_ms, None);
6488        assert_eq!(health.last_action.as_deref(), Some("restart"));
6489        assert_eq!(health.last_action_ms, Some(124));
6490    }
6491
6492    #[cfg(unix)]
6493    #[tokio::test]
6494    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6495        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6496        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6497        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6498        drop(unused);
6499        let configured = http_fixture(&dir, &url, 2);
6500        let module =
6501            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6502                .supervise_configured_with_health(
6503                    configured.module_spec(),
6504                    true,
6505                    configured.health,
6506                    configured.drain_timeout_ms,
6507                    configured.restart,
6508                )
6509                .unwrap();
6510        let before = module.status().unwrap().spawn_generation;
6511        timeout(Duration::from_secs(5), async {
6512            loop {
6513                let status = module.status().unwrap();
6514                if status.spawn_generation > before {
6515                    assert!(status.lifetime_restarts > 0);
6516                    break;
6517                }
6518                sleep(Duration::from_millis(5)).await;
6519            }
6520        })
6521        .await
6522        .expect("sustained HTTP refusal must trigger the health restart policy");
6523        module.drain().await.unwrap();
6524    }
6525
6526    #[cfg(unix)]
6527    #[tokio::test]
6528    async fn http_health_timeout_honours_deadline() {
6529        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6530        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6531        let server = tokio::spawn(async move {
6532            let _held = listener.accept().await.unwrap();
6533            std::future::pending::<()>().await;
6534        });
6535        let error = timeout(
6536            Duration::from_secs(1),
6537            probe_http_health(&url, Duration::from_millis(10)),
6538        )
6539        .await
6540        .expect("the probe must enforce its own deadline")
6541        .unwrap_err();
6542        server.abort();
6543        assert!(error.to_string().contains("timed out"));
6544    }
6545
6546    #[cfg(unix)]
6547    #[tokio::test]
6548    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6549        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6550        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6551        let url = format!(
6552            "http://localhost:{}/healthz",
6553            listener.local_addr().unwrap().port()
6554        );
6555        let server = tokio::spawn(async move {
6556            let (mut stream, _) = listener.accept().await.unwrap();
6557            let mut request = [0u8; 2048];
6558            assert!(stream.read(&mut request).await.unwrap() > 0);
6559            stream
6560                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6561                .await
6562                .unwrap();
6563        });
6564        // The deadline only bounds a hang. A probe that never tried the IPv6
6565        // address would be refused on 127.0.0.1 and fail at once, so a longer
6566        // deadline does not weaken the assertion; one second timed out under a
6567        // loaded parallel test run.
6568        assert_eq!(
6569            probe_http_health(&url, Duration::from_secs(10))
6570                .await
6571                .unwrap()
6572                .status,
6573            HealthStatus::Ok
6574        );
6575        server.await.unwrap();
6576    }
6577
6578    #[cfg(unix)]
6579    #[tokio::test]
6580    async fn http_health_timeout_keeps_partial_status_and_body() {
6581        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6582        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6583        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6584        let server = tokio::spawn(async move {
6585            let (mut stream, _) = listener.accept().await.unwrap();
6586            let mut request = [0u8; 2048];
6587            assert!(stream.read(&mut request).await.unwrap() > 0);
6588            stream
6589                .write_all(
6590                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6591                )
6592                .await
6593                .unwrap();
6594            std::future::pending::<()>().await;
6595        });
6596        let error = probe_http_health(&url, Duration::from_secs(1))
6597            .await
6598            .unwrap_err()
6599            .to_string();
6600        server.abort();
6601        assert!(
6602            error.contains("timed out")
6603                && error.contains("503 Unavailable")
6604                && error.contains("partial diagnostic"),
6605            "{error}"
6606        );
6607    }
6608
6609    #[cfg(unix)]
6610    #[tokio::test]
6611    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6612        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6613        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6614        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6615        let server = tokio::spawn(async move {
6616            let (mut stream, _) = listener.accept().await.unwrap();
6617            let mut request = [0u8; 2048];
6618            assert!(stream.read(&mut request).await.unwrap() > 0);
6619            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6620            let response = format!(
6621                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6622                body.len()
6623            );
6624            stream.write_all(response.as_bytes()).await.unwrap();
6625        });
6626        let error = probe_http_health(&url, Duration::from_secs(1))
6627            .await
6628            .unwrap_err()
6629            .to_string();
6630        server.await.unwrap();
6631        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6632        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6633        assert!(!error.contains("not-in-diagnostic"));
6634    }
6635
6636    #[cfg(unix)]
6637    #[tokio::test]
6638    async fn http_health_real_nats_server_monitoring() {
6639        if std::process::Command::new("nats-server")
6640            .arg("--version")
6641            .env("XDG_DATA_HOME", std::env::temp_dir())
6642            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6643            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6644            .output()
6645            .is_err()
6646        {
6647            eprintln!(
6648                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6649            );
6650            return;
6651        }
6652        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6653        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6654        let port = monitor.local_addr().unwrap().port();
6655        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6656        let client_port = client.local_addr().unwrap().port();
6657        let config = dir.join("server.conf");
6658        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6659        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6660        configured.program = PathBuf::from("nats-server");
6661        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6662        drop(monitor);
6663        drop(client);
6664        let module =
6665            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6666                .supervise_configured_with_health(
6667                    configured.module_spec(),
6668                    true,
6669                    configured.health,
6670                    configured.drain_timeout_ms,
6671                    configured.restart,
6672                )
6673                .unwrap();
6674        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6675        module.drain().await.unwrap();
6676    }
6677
6678    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6679    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6680        let supervisor =
6681            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6682        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6683        let initial = ModuleSpec {
6684            module_id: "rescan-preserves-spawn-facts".to_string(),
6685            program: PathBuf::from("/spawned/module"),
6686            args: Vec::new(),
6687            env: Vec::new(),
6688            reserved: false,
6689            reserved_prefixes: Vec::new(),
6690            protocol: ModuleProtocol::Subc,
6691            overlap: Default::default(),
6692        };
6693        let module = supervisor.supervised_module(
6694            initial.clone(),
6695            supervisor.runtime_config(),
6696            snapshot,
6697            None,
6698        );
6699        let before = module.status().unwrap();
6700        let mut replacement = initial;
6701        replacement.program = PathBuf::from("/rescanned/replacement-module");
6702
6703        module
6704            .update_configuration(replacement, HealthConfig::default(), None)
6705            .await
6706            .unwrap();
6707
6708        let after = module.status().unwrap();
6709        assert_eq!(after.pid, before.pid);
6710        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6711        assert_eq!(after.spawned_from, before.spawned_from);
6712        drop(module);
6713    }
6714}
6715
6716fn unix_ms_now() -> u64 {
6717    SystemTime::now()
6718        .duration_since(UNIX_EPOCH)
6719        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6720        .unwrap_or(0)
6721}
6722
6723async fn supervise_loop(
6724    mut spec: ModuleSpec,
6725    mut runtime: SupervisorRuntimeConfig,
6726    registry: Arc<Registry>,
6727    process_liveness: Arc<SupervisorProcessLiveness>,
6728    snapshot: SharedSnapshot,
6729    mut child: Option<SupervisedChild>,
6730    mut commands: mpsc::Receiver<SupervisorCommand>,
6731) {
6732    let mut health_probe = HealthProbeRuntime::default();
6733    // Registry writes (including embedded callers) notify this module only.
6734    // Subscribe before the first refresh; watch retains changes that arrive
6735    // while commands or probes are running. No polling fallback is needed.
6736    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6737        Ok(changes) => Some(changes),
6738        Err(err) => {
6739            // A poisoned registry cannot accept further writes, so it cannot
6740            // recover via a timer. Keep exit and command handling alive.
6741            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6742            None
6743        }
6744    };
6745    // All restart backoffs run here, including health and operator requests.
6746    // While one is pending the loop serves commands, so disable or drain can
6747    // cancel the replacement without spawning a process just to stop it.
6748    let mut pending_respawn: Option<PendingRespawn> = None;
6749    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6750    // before anything else so a stop that interrupted a swap runs at once.
6751    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6752    let mut previous_instance_recheck = tokio::time::interval_at(
6753        Instant::now() + Duration::from_secs(5),
6754        Duration::from_secs(5),
6755    );
6756    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6757    loop {
6758        #[cfg(test)]
6759        {
6760            lock_snapshot(&snapshot).unwrap().actor_select = None;
6761        }
6762        #[cfg(target_os = "macos")]
6763        if let Some(active) = child.as_mut() {
6764            active.confirm_privacy_exec().await;
6765        }
6766        if let Some(scheduled) = runtime
6767            .scheduled_respawn
6768            .lock()
6769            .unwrap_or_else(|p| p.into_inner())
6770            .take()
6771        {
6772            runtime.recovery_is_daemon = scheduled.daemon_initiated;
6773            pending_respawn = Some(scheduled);
6774        }
6775        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6776            pending_respawn = None;
6777            cancel_deferred_reload(
6778                &runtime,
6779                &spec.module_id,
6780                "respawn cancelled by a supervisor command",
6781            );
6782        }
6783        if child.is_none()
6784            && pending_respawn.is_none()
6785            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6786        {
6787            cancel_deferred_reload(
6788                &runtime,
6789                &spec.module_id,
6790                "respawn cancelled before a replacement was spawned",
6791            );
6792            let mut stranded = false;
6793            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6794                state.respawn_pending = false;
6795                state.coalesced_restart_pending = false;
6796                if matches!(
6797                    state.state,
6798                    ModuleState::Restarting
6799                        | ModuleState::Starting
6800                        | ModuleState::Draining
6801                        | ModuleState::Unresponsive
6802                ) {
6803                    error!(module_id = %spec.module_id, state = ?state.state,
6804                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6805                    stranded = true;
6806                }
6807            });
6808            if stranded {
6809                fail_snapshot(
6810                    &snapshot,
6811                    Some(&spec.module_id),
6812                    None,
6813                    &runtime.failure_context(runtime.recovery_is_daemon),
6814                );
6815            }
6816        }
6817        if let Some(command) = requeued.pop_front() {
6818            if !handle_supervisor_command(
6819                command,
6820                &mut spec,
6821                &mut runtime,
6822                &registry,
6823                &process_liveness,
6824                &snapshot,
6825                &mut child,
6826                &mut commands,
6827                &mut requeued,
6828            )
6829            .await
6830            {
6831                return;
6832            }
6833            if child.is_some() || !respawn_still_pending(&snapshot) {
6834                pending_respawn = None;
6835                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6836                    state.respawn_pending = false
6837                });
6838            }
6839            continue;
6840        }
6841        if child.is_some() {
6842            runtime.recovery_is_daemon = true;
6843            #[cfg(test)]
6844            {
6845                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6846            }
6847            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6848            let no_checks = runtime.health.http.is_none()
6849                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6850                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6851            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6852                state
6853                    .failed_retry
6854                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
6855            });
6856            let wake_after = match (health_probe.wake_after(), uptime_reset) {
6857                (probe, Some(reset)) => Some(
6858                    probe
6859                        .unwrap_or(Duration::MAX)
6860                        .min(reset.saturating_duration_since(Instant::now())),
6861                ),
6862                (probe, None) => probe,
6863            };
6864            let probe_sleep = async move {
6865                match wake_after {
6866                    Some(delay) => sleep(delay).await,
6867                    None => std::future::pending().await,
6868                }
6869            };
6870            tokio::pin!(probe_sleep);
6871            // A `protocol: "none"` child (a plain process that never registers
6872            // over the subc wire, such as nats-server) with no HTTP health
6873            // check has nothing to probe, so it waits only for its exit or a
6874            // supervisor command. A rescan that adds an HTTP check is a command.
6875            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6876            #[cfg(test)]
6877            {
6878                let registration_pending = wire_child
6879                    && registration_changes
6880                        .as_ref()
6881                        .is_some_and(|changes| changes.has_changed().unwrap());
6882                let mut state = lock_snapshot(&snapshot).unwrap();
6883                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6884                    generation: state.spawn_generation,
6885                    turn: state.actor_turns,
6886                    registered_connection: health_probe.registered_connection,
6887                    next_probe_at: health_probe.next_probe_at,
6888                    wake_after,
6889                });
6890            }
6891            let active_child = child.as_mut().expect("child checked above");
6892            tokio::select! {
6893                wait_result = active_child.wait() => {
6894                    if let Ok(mut state) = lock_snapshot(&snapshot) {
6895                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6896                    }
6897                    #[cfg(test)]
6898                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6899                    // Every arm below that gives up on the CHILD must keep the
6900                    // supervision task itself alive (child = None, loop
6901                    // continues into command-serving mode). Returning here
6902                    // closes the command channel, which makes the module
6903                    // permanently unrestartable in-band: a clean child exit
6904                    // of an enabled module once wedged the fleet this way
6905                    // ('supervisor command channel is closed') and required a
6906                    // full daemon restart to recover.
6907                    let exit_report = match wait_result {
6908                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6909                        Err(err) => {
6910                            active_child.drain_stderr(&spec.module_id).await;
6911                            fail_after_wait_error(
6912                                &spec.module_id,
6913                                &snapshot,
6914                                &runtime.terminal_ring,
6915                                &runtime.spawn_events,
6916                                format!("failed to wait for supervised module: {err}"),
6917                            &runtime.failure_context(true),
6918                            );
6919                            untrack_if_registration_released(
6920                                &process_liveness,
6921                                &registry,
6922                                &spec.module_id,
6923                                &snapshot,
6924                            );
6925                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6926                            child = None;
6927                            continue;
6928                        }
6929                    };
6930                    active_child.drain_stderr(&spec.module_id).await;
6931
6932                    let next = on_child_exit(
6933                        &spec,
6934                        runtime.restart_policy,
6935                        &registry,
6936                        &snapshot,
6937                        &runtime.terminal_ring,
6938                        &runtime.spawn_events,
6939                        &runtime.child_roster,
6940                        exit_report,
6941                        true,
6942                    ).await;
6943                    runtime.start_requester = None;
6944                    runtime.pending_activity_restart = false;
6945                    // The exit is recorded, so a daemon shutdown may stop
6946                    // waiting for this child (see `SupervisedChild::wait`).
6947                    active_child.release_roster();
6948                    match next {
6949                        NextAction::Stop { registration_released } => {
6950                            if registration_released {
6951                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6952                            }
6953                            child = None;
6954                        }
6955                        NextAction::Restart { schedule } => {
6956                            let delay = schedule.map_or(
6957                                runtime.restart_policy.delay_for_restart(0),
6958                                |schedule| schedule.delay,
6959                            );
6960                            if let Some(schedule) = schedule {
6961                                log_crash_respawn(&spec.module_id, schedule);
6962                            }
6963                            // The exited child is fully recorded at this point,
6964                            // so release it and count the backoff down in the
6965                            // command-serving branch below rather than sleeping
6966                            // here: commands cannot be received from inside this
6967                            // select arm, and an operator disable or drain that
6968                            // arrives during the backoff must cancel the pending
6969                            // respawn instead of waiting for it to spawn first.
6970                            child = None;
6971                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
6972                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6973                        }
6974                    }
6975                }
6976                command = commands.recv() => {
6977                    #[cfg(test)]
6978                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6979                    let Some(command) = command else {
6980                        return;
6981                    };
6982                    if !handle_supervisor_command(
6983                        command,
6984                        &mut spec,
6985                        &mut runtime,
6986                        &registry,
6987                        &process_liveness,
6988                        &snapshot,
6989                        &mut child,
6990                        &mut commands,
6991                        &mut requeued,
6992                    ).await {
6993                        return;
6994                    }
6995                }
6996                _ = async {
6997                    match registration_changes.as_mut() {
6998                        Some(changes) => { let _ = changes.changed().await; }
6999                        None => std::future::pending().await,
7000                    }
7001                }, if wire_child => {
7002                    #[cfg(test)]
7003                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7004                }
7005                _ = &mut probe_sleep => {
7006                    #[cfg(test)]
7007                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7008                    if health_probe.due() {
7009                        run_health_probe_cycle(
7010                            &spec,
7011                            &runtime,
7012                            &registry,
7013                            &process_liveness,
7014                            &snapshot,
7015                            &mut child,
7016                        ).await;
7017                        if child.is_none() {
7018                            runtime.start_requester = None;
7019                            runtime.pending_activity_restart = false;
7020                        }
7021                        if child.is_some() {
7022                            health_probe.schedule_next(&spec, runtime.health.cadence);
7023                        }
7024                    }
7025                }
7026            }
7027        } else if let Some(pending) = pending_respawn {
7028            tokio::select! {
7029                _ = sleep_until(pending.deadline) => {
7030                    pending_respawn = None;
7031                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7032                    // A command handled below while the backoff elapsed may
7033                    // have stopped the module; never respawn past an operator's
7034                    // disable or drain.
7035                    if !respawn_still_pending(&snapshot) {
7036                        continue;
7037                    }
7038                    // The daemon began shutting down during the backoff: the
7039                    // spawn would be refused anyway, and refusing it here
7040                    // leaves the module stopped instead of reporting a
7041                    // failed restart.
7042                    if runtime.child_roster.is_closed() {
7043                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7044                            state.state = ModuleState::Stopped;
7045                        });
7046                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7047                        continue;
7048                    }
7049                    if let Err(err) = release_dead_registration(
7050                        &registry,
7051                        runtime.forwarding.as_deref(),
7052                        &snapshot,
7053                        &spec.module_id,
7054                        &runtime.failure_context(runtime.recovery_is_daemon),
7055                    ).await {
7056                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7057                        continue;
7058                    }
7059
7060                    if matches!(pending.kind, RespawnKind::Reload) {
7061                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7062                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7063                        if result.is_ok() && runtime.pending_activity_restart {
7064                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7065                        }
7066                        runtime.pending_activity_restart = false;
7067                        if let Some(reply) = reply { let _ = reply.send(result); }
7068                        continue;
7069                    }
7070                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7071                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7072                        Ok(next_child) => {
7073                            child = Some(next_child);
7074                            if runtime.pending_activity_restart {
7075                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7076                                runtime.pending_activity_restart = false;
7077                            }
7078                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7079                        }
7080                        Err(err) => {
7081                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7082                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7083                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7084                        }
7085                    }
7086                }
7087                command = commands.recv() => {
7088                    let Some(command) = command else {
7089                        return;
7090                    };
7091                    if !handle_supervisor_command(
7092                        command,
7093                        &mut spec,
7094                        &mut runtime,
7095                        &registry,
7096                        &process_liveness,
7097                        &snapshot,
7098                        &mut child,
7099                        &mut commands,
7100                        &mut requeued,
7101                    ).await {
7102                        return;
7103                    }
7104                    // Reconcile the pending respawn with what the command did:
7105                    // a start may already have spawned a fresh child,
7106                    // while a disable or drain moved the snapshot out of the
7107                    // state the respawn was counting down from.
7108                    if child.is_some() || !respawn_still_pending(&snapshot) {
7109                        pending_respawn = None;
7110                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7111                    }
7112                }
7113            }
7114        } else {
7115            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7116                (state.enabled && state.state == ModuleState::Failed)
7117                    .then_some(state.failed_retry.pending)
7118                    .flatten()
7119            });
7120            let command = if lock_snapshot(&snapshot)
7121                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7122            {
7123                tokio::select! {
7124                    command = commands.recv() => command,
7125                    _ = previous_instance_recheck.tick() => {
7126                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7127                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7128                            if enabled && !runtime.child_roster.is_closed() {
7129                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7130                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7131                                    Ok(next_child) => child = Some(next_child),
7132                                    Err(error) => {
7133                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7134                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7135                                    }
7136                                }
7137                            } else {
7138                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7139                                    state.waiting_for_previous_pid = None;
7140                                    state.waiting_for_previous_detail = None;
7141                                });
7142                            }
7143                        }
7144                        continue;
7145                    }
7146                }
7147            } else {
7148                tokio::select! {
7149                    biased;
7150                    command = commands.recv() => command,
7151                    _ = async {
7152                        match slow_retry {
7153                            Some(retry) => sleep_until(retry.deadline).await,
7154                            None => std::future::pending().await,
7155                        }
7156                    } => {
7157                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7158                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7159                            state.failed_retry.pending = None;
7160                        });
7161                        if runtime.child_roster.is_closed() {
7162                            continue;
7163                        }
7164                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7165                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7166                        if let Err(error) = set_child_enabled(
7167                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7168                            &runtime.failure_context(true),
7169                        ).await {
7170                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7171                        }
7172                        continue;
7173                    }
7174                }
7175            };
7176            let Some(command) = command else {
7177                return;
7178            };
7179            if !handle_supervisor_command(
7180                command,
7181                &mut spec,
7182                &mut runtime,
7183                &registry,
7184                &process_liveness,
7185                &snapshot,
7186                &mut child,
7187                &mut commands,
7188                &mut requeued,
7189            )
7190            .await
7191            {
7192                return;
7193            }
7194        }
7195    }
7196}
7197
7198fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7199    info!(
7200        module_id,
7201        restart_in_window = schedule.restart_in_window,
7202        delay_ms = schedule.delay.as_millis() as u64,
7203        "respawning after crash"
7204    );
7205}
7206
7207/// Whether the respawn a backoff was counting down to is still wanted. A
7208/// disable or drain handled while the backoff elapsed moves the snapshot out
7209/// of `Restarting`, and the operator's stop must win over the pending respawn,
7210/// so every sleep-then-spawn path re-validates against the live snapshot
7211/// instead of assuming the state it left behind still holds.
7212fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7213    matches!(
7214        lock_snapshot(snapshot),
7215        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7216    )
7217}
7218
7219enum NextAction {
7220    Stop {
7221        registration_released: bool,
7222    },
7223    Restart {
7224        schedule: Option<CrashRestartSchedule>,
7225    },
7226}
7227
7228#[allow(clippy::too_many_arguments)]
7229async fn handle_supervisor_command(
7230    command: SupervisorCommand,
7231    spec: &mut ModuleSpec,
7232    runtime: &mut SupervisorRuntimeConfig,
7233    registry: &Arc<Registry>,
7234    process_liveness: &SupervisorProcessLiveness,
7235    snapshot: &SharedSnapshot,
7236    child: &mut Option<SupervisedChild>,
7237    commands: &mut mpsc::Receiver<SupervisorCommand>,
7238    requeued: &mut VecDeque<SupervisorCommand>,
7239) -> bool {
7240    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7241        runtime.recovery_is_daemon = false;
7242    }
7243    if matches!(
7244        command,
7245        SupervisorCommand::Drain { .. }
7246            | SupervisorCommand::Retire { .. }
7247            | SupervisorCommand::SetEnabled { .. }
7248            | SupervisorCommand::Restart { .. }
7249    ) {
7250        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7251            state.failed_retry.cancel()
7252        });
7253    }
7254    let command =
7255        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7256            match command {
7257                SupervisorCommand::Restart { reply, .. }
7258                | SupervisorCommand::Reload { reply, .. }
7259                | SupervisorCommand::Swap { reply, .. } => {
7260                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7261                    return true;
7262                }
7263                SupervisorCommand::SetEnabled {
7264                    enabled: true,
7265                    reply,
7266                    ..
7267                } => {
7268                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7269                    return true;
7270                }
7271                other => other,
7272            }
7273        } else {
7274            command
7275        };
7276    match command {
7277        SupervisorCommand::Drain { requester, reply } => {
7278            // A plain stop runs no forwarding drain, so nothing reaches the
7279            // module over its connection before the wait: ask by signal.
7280            let result = drain_optional_child(
7281                &spec.module_id,
7282                spec.protocol,
7283                StopNotice::NotSent,
7284                registry,
7285                runtime.forwarding.as_deref(),
7286                snapshot,
7287                &runtime.terminal_ring,
7288                &runtime.spawn_events,
7289                child,
7290                runtime.drain_timeout,
7291                ModuleState::Stopped,
7292                None,
7293                &FailureContext::operator(),
7294            )
7295            .await;
7296            let registration_released = result.is_ok();
7297            if registration_released {
7298                runtime.record_activity_transition(
7299                    "subc.module_stopped",
7300                    &spec.module_id,
7301                    requester,
7302                );
7303            }
7304            let _ = reply.send(result);
7305            if registration_released {
7306                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7307            }
7308            false
7309        }
7310        SupervisorCommand::Retire { requester, reply } => {
7311            let result = async {
7312                let stop_notice = begin_forwarding_drain_if_configured(
7313                    spec,
7314                    runtime,
7315                    registry,
7316                    snapshot,
7317                    None,
7318                    RouteCloseReason::Disable,
7319                )
7320                .await?;
7321                drain_optional_child(
7322                    &spec.module_id,
7323                    spec.protocol,
7324                    stop_notice,
7325                    registry,
7326                    runtime.forwarding.as_deref(),
7327                    snapshot,
7328                    &runtime.terminal_ring,
7329                    &runtime.spawn_events,
7330                    child,
7331                    runtime.drain_timeout,
7332                    ModuleState::Stopped,
7333                    None,
7334                    &FailureContext::operator(),
7335                )
7336                .await
7337            }
7338            .await;
7339            let registration_released = result.is_ok();
7340            if registration_released {
7341                runtime.record_activity_transition(
7342                    "subc.module_stopped",
7343                    &spec.module_id,
7344                    requester,
7345                );
7346            }
7347            let _ = reply.send(result);
7348            if registration_released {
7349                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7350            }
7351            false
7352        }
7353        SupervisorCommand::Restart {
7354            requester,
7355            drain_timeout_ms,
7356            received_at_generation,
7357            queued_at,
7358            reply,
7359        } => {
7360            // Without this line a restart that waited in the queue (behind a
7361            // health probe cycle or another command) was invisible: the log
7362            // showed only the drain timing out, minutes after the operator's call.
7363            info!(
7364                module_id = %spec.module_id,
7365                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7366                "restart command dequeued"
7367            );
7368            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7369            // caller whose own request lane rides the module being restarted: the
7370            // caller's in-flight request keeps the drain from quiescing, the drain
7371            // keeps the restart from completing, and the completion keeps the reply
7372            // from releasing the caller — so the drain always timed out and cut the
7373            // initiator with a GOODBYE, even on a healthy module. Replying once the
7374            // restart is validated lets a self-lane caller settle, which is exactly
7375            // what makes the drain succeed. Completion is observable via
7376            // supervisor.list / module status; a post-ack failure lands the module
7377            // in a visible terminal state below rather than in a reply nobody can
7378            // receive.
7379            let validation = match lock_snapshot(snapshot) {
7380                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7381                    module_id: spec.module_id.clone(),
7382                }),
7383                Ok(_) => Ok(()),
7384                Err(err) => Err(err),
7385            };
7386            let initiated = validation.is_ok();
7387            let _ = reply.send(validation);
7388            // A restart asks for a fresh process. Commands run one at a time,
7389            // so a restart queued behind another restart (two operator calls
7390            // in quick succession) is dequeued the moment the first one has
7391            // spawned its replacement -- before that process has sent HELLO.
7392            // Running it would drain and kill the process the first restart
7393            // just produced, which is the opposite of what both callers asked
7394            // for. If a process spawned after this request was received is
7395            // still supervised, the request is already satisfied. Not when the
7396            // configuration changed since that spawn: then the newer process
7397            // predates the spec this restart may exist to apply.
7398            // Generation publication can lag a child that is already serving
7399            // while its identity is recorded. Only process creation after request
7400            // receipt satisfies a restart, not publication after receipt.
7401            let satisfied_by_generation = if initiated
7402                && child
7403                    .as_ref()
7404                    .is_some_and(|child| child.spawned_at > queued_at)
7405            {
7406                lock_snapshot(snapshot).ok().and_then(|state| {
7407                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7408                })
7409            } else {
7410                None
7411            };
7412            let satisfied_by_pending = initiated
7413                && child.is_none()
7414                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7415                    // No child exists yet, so this replacement will necessarily
7416                    // be created after the queued request, unlike a published
7417                    // generation whose process may already have been serving.
7418                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7419                    if pending {
7420                        state.coalesced_restart_pending = true;
7421                    }
7422                    pending
7423                });
7424            if satisfied_by_pending {
7425                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7426            } else if let Some(generation) = satisfied_by_generation {
7427                info!(
7428                    module_id = %spec.module_id,
7429                    received_at_generation,
7430                    "restart already satisfied by generation {generation}; not restarting again"
7431                );
7432            } else if initiated {
7433                // Precedence: this restart's operator override, else the module's
7434                // configured budget (already resolved into the runtime).
7435                let drain_timeout = drain_timeout_ms
7436                    .map(Duration::from_millis)
7437                    .unwrap_or(runtime.drain_timeout);
7438                if let Err(err) = restart_child(
7439                    spec,
7440                    runtime,
7441                    registry,
7442                    process_liveness,
7443                    snapshot,
7444                    child,
7445                    drain_timeout,
7446                )
7447                .await
7448                {
7449                    warn!(
7450                        module_id = %spec.module_id,
7451                        error = %err,
7452                        "operator restart failed after initiation ack; module state carries the outcome"
7453                    );
7454                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7455                        state.state = ModuleState::Failed;
7456                        clear_current_process_facts(state);
7457                    });
7458                } else {
7459                    runtime.start_requester = requester;
7460                    runtime.pending_activity_restart = true;
7461                }
7462            }
7463            true
7464        }
7465        SupervisorCommand::Reload { requester, reply } => {
7466            let result =
7467                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7468            if result.is_ok() {
7469                runtime.start_requester = requester;
7470                runtime.pending_activity_restart = true;
7471            }
7472            if result.is_ok()
7473                && runtime
7474                    .scheduled_respawn
7475                    .lock()
7476                    .unwrap_or_else(|p| p.into_inner())
7477                    .as_ref()
7478                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7479            {
7480                *runtime
7481                    .deferred_reload_reply
7482                    .lock()
7483                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7484            } else {
7485                let _ = reply.send(result);
7486            }
7487            true
7488        }
7489        SupervisorCommand::SetEnabled {
7490            enabled,
7491            requester,
7492            reply,
7493        } => {
7494            let previous_requester = runtime.start_requester.clone();
7495            if enabled {
7496                runtime.start_requester = requester.clone();
7497            }
7498            let result = set_child_enabled(
7499                spec,
7500                runtime,
7501                registry,
7502                process_liveness,
7503                snapshot,
7504                child,
7505                enabled,
7506                &runtime.failure_context(false),
7507            )
7508            .await;
7509            if matches!(result, Ok(true)) {
7510                runtime.pending_activity_restart = false;
7511                if enabled {
7512                    runtime.start_requester = requester;
7513                } else {
7514                    runtime.start_requester = None;
7515                    runtime.record_activity_transition(
7516                        "subc.module_disabled",
7517                        &spec.module_id,
7518                        requester,
7519                    );
7520                }
7521            } else {
7522                runtime.start_requester = previous_requester;
7523            }
7524            let _ = reply.send(result);
7525            true
7526        }
7527        SupervisorCommand::UpdateConfiguration {
7528            spec: next_spec,
7529            health,
7530            drain_timeout_ms,
7531            reply,
7532        } => {
7533            if let Some(handle) = &runtime.supervisor_handle {
7534                handle.apply_identity_configuration(&next_spec);
7535            }
7536            *spec = next_spec;
7537            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7538                state.configuration_updated_since_spawn = true;
7539            });
7540            let health_changed = runtime.health != health;
7541            runtime.health = health;
7542            // Reset the cadence and old endpoint's failure streak on a live
7543            // health-policy change rather than waiting for its old deadline.
7544            if health_changed {
7545                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7546                    state.health = ModuleHealthStatus::default();
7547                });
7548            }
7549            runtime.drain_timeout = drain_timeout_ms
7550                .map(Duration::from_millis)
7551                .unwrap_or(runtime.default_drain_timeout);
7552            *runtime
7553                .effective_drain_timeout
7554                .lock()
7555                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7556            let _ = reply.send(());
7557            true
7558        }
7559        SupervisorCommand::Swap {
7560            requester,
7561            ready_timeout,
7562            reply,
7563        } => {
7564            let mut swap_runtime = runtime.clone();
7565            swap_runtime.start_requester = requester;
7566            let before = lock_snapshot(snapshot)
7567                .ok()
7568                .map(|state| state.spawn_generation);
7569            let end = swap::run_swap(
7570                spec,
7571                &swap_runtime,
7572                registry,
7573                process_liveness,
7574                snapshot,
7575                child,
7576                commands,
7577                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7578                reply,
7579            )
7580            .await;
7581            if before
7582                != lock_snapshot(snapshot)
7583                    .ok()
7584                    .map(|state| state.spawn_generation)
7585            {
7586                runtime.start_requester = swap_runtime.start_requester;
7587            }
7588            requeued.extend(end.requeue);
7589            true
7590        }
7591    }
7592}
7593
7594async fn restart_child(
7595    spec: &ModuleSpec,
7596    runtime: &SupervisorRuntimeConfig,
7597    registry: &Registry,
7598    process_liveness: &SupervisorProcessLiveness,
7599    snapshot: &SharedSnapshot,
7600    child: &mut Option<SupervisedChild>,
7601    drain_timeout: Duration,
7602) -> Result<(), SuperviseError> {
7603    // Restart cycles a running module; it must not silently start a disabled one.
7604    if !lock_snapshot(snapshot)?.enabled {
7605        return Err(SuperviseError::Disabled {
7606            module_id: spec.module_id.clone(),
7607        });
7608    }
7609    let stop_notice = begin_forwarding_drain_with_timeout(
7610        spec,
7611        runtime,
7612        registry,
7613        snapshot,
7614        None,
7615        RouteCloseReason::Restart,
7616        drain_timeout,
7617    )
7618    .await?;
7619
7620    if child.is_some() {
7621        drain_optional_child(
7622            &spec.module_id,
7623            spec.protocol,
7624            stop_notice,
7625            registry,
7626            runtime.forwarding.as_deref(),
7627            snapshot,
7628            &runtime.terminal_ring,
7629            &runtime.spawn_events,
7630            child,
7631            drain_timeout,
7632            ModuleState::Restarting,
7633            Some(true),
7634            &FailureContext::operator(),
7635        )
7636        .await?;
7637    } else {
7638        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7639            state.enabled = true;
7640            state.state = ModuleState::Restarting;
7641            clear_current_process_facts(state);
7642        })?;
7643        release_dead_registration(
7644            registry,
7645            runtime.forwarding.as_deref(),
7646            snapshot,
7647            &spec.module_id,
7648            &runtime.failure_context(false),
7649        )
7650        .await?;
7651    }
7652
7653    reset_restart_count(snapshot, &spec.module_id)?;
7654    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7655    schedule_respawn(
7656        runtime,
7657        snapshot,
7658        &spec.module_id,
7659        runtime.restart_policy.backoff,
7660        RespawnKind::Spawn,
7661        false,
7662    )
7663}
7664
7665async fn reload_child(
7666    spec: &ModuleSpec,
7667    runtime: &SupervisorRuntimeConfig,
7668    registry: &Registry,
7669    process_liveness: &SupervisorProcessLiveness,
7670    snapshot: &SharedSnapshot,
7671    child: &mut Option<SupervisedChild>,
7672) -> Result<(), SuperviseError> {
7673    // Reload cycles a running module; it must not silently start a disabled one.
7674    if !lock_snapshot(snapshot)?.enabled {
7675        return Err(SuperviseError::Disabled {
7676            module_id: spec.module_id.clone(),
7677        });
7678    }
7679    let stop_notice = begin_forwarding_drain(
7680        spec,
7681        runtime,
7682        registry,
7683        snapshot,
7684        Some(true),
7685        RouteCloseReason::Reload,
7686    )
7687    .await?;
7688
7689    if child.is_some() {
7690        drain_optional_child(
7691            &spec.module_id,
7692            spec.protocol,
7693            stop_notice,
7694            registry,
7695            runtime.forwarding.as_deref(),
7696            snapshot,
7697            &runtime.terminal_ring,
7698            &runtime.spawn_events,
7699            child,
7700            runtime.drain_timeout,
7701            ModuleState::Restarting,
7702            Some(true),
7703            &FailureContext::operator(),
7704        )
7705        .await?;
7706    } else {
7707        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7708            state.enabled = true;
7709            state.state = ModuleState::Restarting;
7710            clear_current_process_facts(state);
7711        })?;
7712        release_dead_registration(
7713            registry,
7714            runtime.forwarding.as_deref(),
7715            snapshot,
7716            &spec.module_id,
7717            &runtime.failure_context(false),
7718        )
7719        .await?;
7720    }
7721
7722    reset_restart_count(snapshot, &spec.module_id)?;
7723    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7724    schedule_respawn(
7725        runtime,
7726        snapshot,
7727        &spec.module_id,
7728        runtime.restart_policy.backoff,
7729        RespawnKind::Reload,
7730        false,
7731    )
7732}
7733
7734async fn finish_reload_child(
7735    spec: &ModuleSpec,
7736    runtime: &SupervisorRuntimeConfig,
7737    registry: &Registry,
7738    process_liveness: &SupervisorProcessLiveness,
7739    snapshot: &SharedSnapshot,
7740    child: &mut Option<SupervisedChild>,
7741) -> Result<(), SuperviseError> {
7742    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7743    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7744        Ok(next_child) => next_child,
7745        Err(err) => {
7746            return handle_reload_spawn_failure(
7747                spec,
7748                runtime,
7749                process_liveness,
7750                snapshot,
7751                child,
7752                format!("new child failed to spawn: {err}"),
7753            )
7754            .await;
7755        }
7756    };
7757    *child = Some(next_child);
7758
7759    let wait_outcome = {
7760        let active_child = child.as_mut().expect("new reload child was just stored");
7761        wait_for_registration_after_reload(
7762            registry,
7763            &spec.module_id,
7764            snapshot,
7765            active_child,
7766            REGISTRY_RELEASE_TIMEOUT,
7767        )
7768        .await?
7769    };
7770
7771    match wait_outcome {
7772        RegistrationWaitOutcome::Registered => {
7773            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7774            Ok(())
7775        }
7776        RegistrationWaitOutcome::Exited(exit_report) => {
7777            if let Some(active_child) = child.as_mut() {
7778                active_child.drain_stderr(&spec.module_id).await;
7779            }
7780            // Keep the reaped child's roster guard until its terminal is written.
7781            // Shutdown waits on that guard, not on the child Option used for respawn.
7782            let mut exited_child = child.take().expect("exited reload child is still stored");
7783            #[cfg(test)]
7784            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7785                gate.reached.notify_one();
7786                gate.resume.notified().await;
7787            }
7788            let result = handle_reload_child_registration_failure(
7789                spec,
7790                runtime,
7791                registry,
7792                process_liveness,
7793                snapshot,
7794                child,
7795                ReloadRegistrationFailure {
7796                    exit_report: registration_failure_exit_report(exit_report),
7797                    reason: exited_child
7798                        .spawn_failure
7799                        .clone()
7800                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7801                },
7802            )
7803            .await;
7804            exited_child.release_roster();
7805            result
7806        }
7807        RegistrationWaitOutcome::TimedOut => {
7808            let mut timed_out_child = child
7809                .take()
7810                .expect("timed-out reload child is still running");
7811            timed_out_child
7812                .start_kill()
7813                .map_err(|source| SuperviseError::Kill {
7814                    module_id: spec.module_id.clone(),
7815                    source,
7816                })?;
7817            let status = timed_out_child
7818                .wait()
7819                .await
7820                .map_err(|source| SuperviseError::Wait {
7821                    module_id: spec.module_id.clone(),
7822                    source,
7823                })?;
7824            timed_out_child.drain_stderr(&spec.module_id).await;
7825            handle_reload_child_registration_failure(
7826                spec,
7827                runtime,
7828                registry,
7829                process_liveness,
7830                snapshot,
7831                child,
7832                ReloadRegistrationFailure {
7833                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7834                        snapshot,
7835                        &timed_out_child,
7836                        &status,
7837                    )),
7838                    reason: format!(
7839                        "new child did not register within {:?}",
7840                        REGISTRY_RELEASE_TIMEOUT
7841                    ),
7842                },
7843            )
7844            .await
7845        }
7846    }
7847}
7848
7849#[allow(clippy::too_many_arguments)]
7850async fn set_child_enabled(
7851    spec: &ModuleSpec,
7852    runtime: &SupervisorRuntimeConfig,
7853    registry: &Registry,
7854    process_liveness: &SupervisorProcessLiveness,
7855    snapshot: &SharedSnapshot,
7856    child: &mut Option<SupervisedChild>,
7857    enabled: bool,
7858    failure: &FailureContext,
7859) -> Result<bool, SuperviseError> {
7860    let (current_enabled, current_state, respawn_pending) = {
7861        let state = lock_snapshot(snapshot)?;
7862        (state.enabled, state.state, state.respawn_pending)
7863    };
7864    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7865    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7866    // clean (Stopped) has no live process and no other in-band recovery — the
7867    // operator's start is the explicit recovery act and resets the budget. Without
7868    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7869    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7870    // the one providing every agent's shell.
7871    let revive_terminal = enabled
7872        && current_enabled
7873        && child.is_none()
7874        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7875            || (current_state == ModuleState::Restarting && !respawn_pending));
7876    if current_enabled == enabled && !revive_terminal {
7877        return Ok(false);
7878    }
7879
7880    if enabled {
7881        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7882            state.enabled = true;
7883            state.state = ModuleState::Starting;
7884            clear_current_process_facts(state);
7885        })?;
7886        #[cfg(test)]
7887        if runtime.test_seed_stale_facts_before_enable_spawn {
7888            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7889                state.process_alive = true;
7890                state.pid = Some(41);
7891                state.spawned_at_ms = Some(42);
7892                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7893                state.spawned_file_identity = Some(SpawnedFileIdentity {
7894                    device: 43,
7895                    inode: 44,
7896                    #[cfg(windows)]
7897                    windows_image: None,
7898                });
7899            })?;
7900        }
7901        release_dead_registration(
7902            registry,
7903            runtime.forwarding.as_deref(),
7904            snapshot,
7905            &spec.module_id,
7906            failure,
7907        )
7908        .await?;
7909        reset_restart_count(snapshot, &spec.module_id)?;
7910        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7911        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7912            Ok(next_child) => next_child,
7913            Err(err) => {
7914                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7915                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7916                return Err(err);
7917            }
7918        };
7919        *child = Some(next_child);
7920        debug!(module_id = %spec.module_id, "supervised module enabled");
7921        Ok(true)
7922    } else {
7923        let stop_notice = begin_forwarding_drain_if_configured(
7924            spec,
7925            runtime,
7926            registry,
7927            snapshot,
7928            Some(false),
7929            RouteCloseReason::Disable,
7930        )
7931        .await?;
7932        drain_optional_child(
7933            &spec.module_id,
7934            spec.protocol,
7935            stop_notice,
7936            registry,
7937            runtime.forwarding.as_deref(),
7938            snapshot,
7939            &runtime.terminal_ring,
7940            &runtime.spawn_events,
7941            child,
7942            runtime.drain_timeout,
7943            ModuleState::Disabled,
7944            Some(false),
7945            failure,
7946        )
7947        .await?;
7948        debug!(module_id = %spec.module_id, "supervised module disabled");
7949        Ok(true)
7950    }
7951}
7952
7953#[allow(clippy::too_many_arguments)]
7954async fn on_child_exit(
7955    spec: &ModuleSpec,
7956    policy: RestartPolicy,
7957    registry: &Registry,
7958    snapshot: &SharedSnapshot,
7959    terminal_ring: &Arc<Mutex<TerminalRing>>,
7960    spawn_events: &SpawnEventFeed,
7961    roster: &ChildRoster,
7962    exit_report: ExitReport,
7963    daemon_initiated: bool,
7964) -> NextAction {
7965    // Once the daemon has begun shutting down, no exit is a crash to recover
7966    // from: the module is exiting because the daemon is going away (EOF on its
7967    // connection, or a service manager signalling the whole cgroup). Record it
7968    // as such and never schedule a respawn, which would only start a process
7969    // for the shutdown to end again.
7970    if roster.is_closed() {
7971        return on_child_exit_during_daemon_shutdown(
7972            spec,
7973            registry,
7974            snapshot,
7975            terminal_ring,
7976            spawn_events,
7977            exit_report,
7978        )
7979        .await;
7980    }
7981    // Every stop the supervisor itself asks for (operator stop, disable,
7982    // restart, reload, swap, a health restart, a drain that runs out of budget)
7983    // takes the child out of the supervise loop and reaps it in
7984    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7985    // that reaches this point was not requested by the daemon.
7986    //
7987    // For a subc-wire module a clean exit is still a stop: those modules are
7988    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7989    // crash, and exiting 0 is a deliberate choice the module made. A
7990    // `protocol: "none"` module is a stock program we cannot change, and many
7991    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7992    // stop would leave the module down for good after any stray signal, so it
7993    // goes through the crash path instead: it spends restart budget, respawns
7994    // with the crash backoff, and ends `failed` when the budget runs out.
7995    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7996        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7997    match exit_report.kind {
7998        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7999            info!(
8000                module_id = %spec.module_id,
8001                exit_code = ?exit_report.code,
8002                exit_signal = ?exit_report.signal,
8003                "supervised module exited cleanly"
8004            );
8005            if let Err(err) = publish_after_terminal_record(
8006                &spec.module_id,
8007                || {
8008                    record_terminal(
8009                        &spec.module_id,
8010                        terminal_ring,
8011                        spawn_events,
8012                        &exit_report,
8013                        TerminalDisposition::Stopped,
8014                    )
8015                },
8016                || {
8017                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8018                        state.state = ModuleState::Stopped;
8019                        clear_current_process_facts(state);
8020                        state.last_exit = Some(exit_report.clone());
8021                    })
8022                },
8023            ) {
8024                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8025            }
8026            let registration_released = match wait_for_registration_release(
8027                registry,
8028                &spec.module_id,
8029                REGISTRY_RELEASE_TIMEOUT,
8030            )
8031            .await
8032            {
8033                Ok(()) => true,
8034                Err(err) => {
8035                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8036                    false
8037                }
8038            };
8039            NextAction::Stop {
8040                registration_released,
8041            }
8042        }
8043        ExitKind::Clean | ExitKind::Crash => {
8044            if unrequested_clean_exit_of_protocol_none {
8045                warn!(
8046                    module_id = %spec.module_id,
8047                    exit_code = ?exit_report.code,
8048                    exit_signal = ?exit_report.signal,
8049                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8050                );
8051            } else {
8052                warn!(
8053                    module_id = %spec.module_id,
8054                    exit_code = ?exit_report.code,
8055                    exit_signal = ?exit_report.signal,
8056                    "supervised module exited abnormally (crash)"
8057                );
8058            }
8059            let mut restart_schedule = None;
8060            let mut disposition = TerminalDisposition::Disabled;
8061            let mut final_state = ModuleState::Disabled;
8062            // Set only when the budget is what stopped the module, so the
8063            // terminal record says which limit was hit rather than leaving
8064            // `failed` to be read as "crashed once, badly".
8065            let mut disposition_detail = lock_snapshot(snapshot)
8066                .ok()
8067                .and_then(|mut state| state.spawn_failure.take());
8068            let now = Instant::now();
8069            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8070                if state.enabled {
8071                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8072                        // Reserving the last slot exhausts the budget for the
8073                        // replacement, not for this exit. Publish its scheduled
8074                        // recovery atomically with the reservation so connection
8075                        // cleanup sees it while terminal recording still holds
8076                        // the visible state at Running.
8077                        state.respawn_pending = true;
8078                        final_state = ModuleState::Restarting;
8079                        restart_schedule = Some(schedule);
8080                        disposition = TerminalDisposition::Restarting;
8081                    } else {
8082                        final_state = ModuleState::Failed;
8083                        disposition = TerminalDisposition::Failed;
8084                        let budget = policy.budget_exhausted_detail();
8085                        disposition_detail =
8086                            Some(disposition_detail.take().map_or_else(
8087                                || budget.clone(),
8088                                |cause| format!("{cause}; {budget}"),
8089                            ));
8090                    }
8091                } else {
8092                    disposition = TerminalDisposition::Disabled;
8093                }
8094            }) {
8095                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8096                return NextAction::Stop {
8097                    registration_released: false,
8098                };
8099            }
8100            if disposition == TerminalDisposition::Failed {
8101                // The window is in the message, not only in the fields: this line
8102                // is read in a scrollback where a bare `max_restarts=3` reads as a
8103                // lifetime cap and sends the operator looking for three crashes
8104                // that never happened together.
8105                error!(
8106                    module_id = %spec.module_id,
8107                    max_restarts = policy.max_restarts,
8108                    window_secs = policy.window.as_secs(),
8109                    "module stopped: {}",
8110                    policy.budget_exhausted_detail()
8111                );
8112            }
8113            if let Err(err) = publish_after_terminal_record(
8114                &spec.module_id,
8115                || {
8116                    record_terminal_with_detail(
8117                        &spec.module_id,
8118                        terminal_ring,
8119                        spawn_events,
8120                        &exit_report,
8121                        disposition,
8122                        disposition_detail,
8123                    )
8124                },
8125                || {
8126                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8127                        state.state = final_state;
8128                        clear_current_process_facts(state);
8129                        if final_state == ModuleState::Failed
8130                            && daemon_initiated
8131                            && !roster.is_closed()
8132                        {
8133                            state.failed_retry.schedule(&spec.module_id);
8134                        }
8135                        state.last_exit = Some(exit_report.clone());
8136                    })
8137                },
8138            ) {
8139                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8140            }
8141
8142            if let Some(schedule) = restart_schedule {
8143                NextAction::Restart {
8144                    schedule: Some(schedule),
8145                }
8146            } else {
8147                let registration_released = match wait_for_registration_release(
8148                    registry,
8149                    &spec.module_id,
8150                    REGISTRY_RELEASE_TIMEOUT,
8151                )
8152                .await
8153                {
8154                    Ok(()) => true,
8155                    Err(err) => {
8156                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8157                        false
8158                    }
8159                };
8160                NextAction::Stop {
8161                    registration_released,
8162                }
8163            }
8164        }
8165        ExitKind::DeliberateSeverance => {
8166            warn!(
8167                module_id = %spec.module_id,
8168                exit_code = ?exit_report.code,
8169                exit_signal = ?exit_report.signal,
8170                "supervised module exited after deliberate connection severance"
8171            );
8172            let should_restart = match lock_snapshot(snapshot) {
8173                Ok(state) => state.enabled,
8174                Err(err) => {
8175                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8176                    return NextAction::Stop {
8177                        registration_released: false,
8178                    };
8179                }
8180            };
8181            let final_state = if should_restart {
8182                ModuleState::Restarting
8183            } else {
8184                ModuleState::Disabled
8185            };
8186            if let Err(err) = publish_after_terminal_record(
8187                &spec.module_id,
8188                || {
8189                    record_terminal(
8190                        &spec.module_id,
8191                        terminal_ring,
8192                        spawn_events,
8193                        &exit_report,
8194                        terminal_disposition(final_state),
8195                    )
8196                },
8197                || {
8198                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8199                        state.state = final_state;
8200                        clear_current_process_facts(state);
8201                        state.last_exit = Some(exit_report.clone());
8202                        state.lifetime_restarts += 1;
8203                    })
8204                },
8205            ) {
8206                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8207                return NextAction::Stop {
8208                    registration_released: false,
8209                };
8210            }
8211
8212            if should_restart {
8213                NextAction::Restart { schedule: None }
8214            } else {
8215                let registration_released = match wait_for_registration_release(
8216                    registry,
8217                    &spec.module_id,
8218                    REGISTRY_RELEASE_TIMEOUT,
8219                )
8220                .await
8221                {
8222                    Ok(()) => true,
8223                    Err(err) => {
8224                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8225                        false
8226                    }
8227                };
8228                NextAction::Stop {
8229                    registration_released,
8230                }
8231            }
8232        }
8233    }
8234}
8235
8236async fn on_child_exit_during_daemon_shutdown(
8237    spec: &ModuleSpec,
8238    registry: &Registry,
8239    snapshot: &SharedSnapshot,
8240    terminal_ring: &Arc<Mutex<TerminalRing>>,
8241    spawn_events: &SpawnEventFeed,
8242    exit_report: ExitReport,
8243) -> NextAction {
8244    info!(
8245        module_id = %spec.module_id,
8246        exit_code = ?exit_report.code,
8247        exit_signal = ?exit_report.signal,
8248        exit_kind = ?exit_report.kind,
8249        "supervised module exited during daemon shutdown; not restarting it"
8250    );
8251    if let Err(err) = publish_after_terminal_record(
8252        &spec.module_id,
8253        || {
8254            record_terminal(
8255                &spec.module_id,
8256                terminal_ring,
8257                spawn_events,
8258                &exit_report,
8259                TerminalDisposition::DaemonShutdown,
8260            )
8261        },
8262        || {
8263            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8264                state.state = ModuleState::Stopped;
8265                clear_current_process_facts(state);
8266                state.last_exit = Some(exit_report.clone());
8267            })
8268        },
8269    ) {
8270        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8271    }
8272    let registration_released =
8273        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8274            .await
8275            .is_ok();
8276    NextAction::Stop {
8277        registration_released,
8278    }
8279}
8280
8281// Keep the status lock out of terminal I/O. Readers may observe the previous
8282// state while recording is in progress, but never a completed exit without its
8283// history. Journal errors are logged by the journal and retained in the ring;
8284// a recording panic still publishes the final state before resuming the unwind.
8285fn publish_after_terminal_record<T>(
8286    module_id: &str,
8287    record: impl FnOnce(),
8288    publish: impl FnOnce() -> T,
8289) -> T {
8290    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8291    let published = publish();
8292    if let Err(panic) = recorded {
8293        error!(
8294            module_id,
8295            "terminal recording panicked; published final state before resuming unwind"
8296        );
8297        std::panic::resume_unwind(panic);
8298    }
8299    published
8300}
8301
8302fn fail_after_wait_error(
8303    module_id: &str,
8304    snapshot: &SharedSnapshot,
8305    terminal_ring: &Arc<Mutex<TerminalRing>>,
8306    spawn_events: &SpawnEventFeed,
8307    detail: String,
8308    failure: &FailureContext,
8309) {
8310    let report = wait_error_exit_report();
8311    publish_after_terminal_record(
8312        module_id,
8313        || {
8314            record_terminal_with_detail(
8315                module_id,
8316                terminal_ring,
8317                spawn_events,
8318                &report,
8319                TerminalDisposition::Failed,
8320                Some(detail),
8321            )
8322        },
8323        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8324    );
8325}
8326
8327#[cfg(test)]
8328fn record_wait_error_terminal(
8329    module_id: &str,
8330    terminal_ring: &Arc<Mutex<TerminalRing>>,
8331    spawn_events: &SpawnEventFeed,
8332) {
8333    record_terminal(
8334        module_id,
8335        terminal_ring,
8336        spawn_events,
8337        &wait_error_exit_report(),
8338        TerminalDisposition::Failed,
8339    );
8340}
8341
8342fn record_terminal(
8343    module_id: &str,
8344    terminal_ring: &Arc<Mutex<TerminalRing>>,
8345    spawn_events: &SpawnEventFeed,
8346    exit_report: &ExitReport,
8347    disposition: TerminalDisposition,
8348) {
8349    record_terminal_with_detail(
8350        module_id,
8351        terminal_ring,
8352        spawn_events,
8353        exit_report,
8354        disposition,
8355        None,
8356    );
8357}
8358
8359/// The ring lock is held only to capture the read (see
8360/// `TerminalJournal::capture_read`), so this module's exits keep recording
8361/// while the journal files are read. Blocking: it reads files.
8362fn durable_terminal_history_of(
8363    terminal_ring: &Mutex<TerminalRing>,
8364    module_id: &str,
8365) -> subc_control::TerminalHistory {
8366    let read = terminal_ring
8367        .lock()
8368        .unwrap_or_else(|p| p.into_inner())
8369        .capture_durable_history();
8370    read.read(module_id)
8371}
8372
8373fn record_terminal_with_detail(
8374    module_id: &str,
8375    terminal_ring: &Arc<Mutex<TerminalRing>>,
8376    spawn_events: &SpawnEventFeed,
8377    exit_report: &ExitReport,
8378    disposition: TerminalDisposition,
8379    disposition_detail: Option<String>,
8380) {
8381    #[cfg(test)]
8382    terminal_history_tests::before_terminal_record(spawn_events);
8383    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8384    let record = TerminalRecord {
8385        exit_code: exit_report.code,
8386        exit_signal: exit_report.signal,
8387        at_ms: exit_report.at_ms,
8388        disposition,
8389        exit_kind: exit_report.kind.into(),
8390        disposition_detail,
8391    };
8392    terminal_ring
8393        .lock()
8394        .unwrap_or_else(|poisoned| poisoned.into_inner())
8395        .record_exit(module_id, record);
8396}
8397
8398fn untrack_if_registration_released(
8399    process_liveness: &SupervisorProcessLiveness,
8400    registry: &Registry,
8401    module_id: &str,
8402    snapshot: &SharedSnapshot,
8403) {
8404    match registry.get_module(module_id) {
8405        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8406        Ok(Some(_)) => {}
8407        Err(err) => {
8408            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8409        }
8410    }
8411}
8412
8413/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8414/// then apply the module's configured entries minus daemon-private capture keys.
8415///
8416/// Separated from `spawn_child` only so it can be asserted without spawning a
8417/// process — a duplicate of this logic in a test would pass while the real one
8418/// drifted, which is the defect class this function exists to avoid.
8419/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8420/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8421/// either and the argument would stop a stock binary from starting at all.
8422/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8423///
8424/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8425/// through [`apply_wire_spawn_args_for_role`].
8426#[cfg(test)]
8427fn apply_wire_spawn_args(
8428    command: &mut Command,
8429    spec: &ModuleSpec,
8430    connection_file_path: Option<&std::path::Path>,
8431    handle: Option<&SupervisorHandle>,
8432) -> Result<Option<NonceHandoff>, SuperviseError> {
8433    apply_wire_spawn_args_for_role(
8434        command,
8435        spec,
8436        connection_file_path,
8437        handle,
8438        SpawnRole::Plain,
8439    )
8440}
8441
8442/// The read end of a spawn's launch-nonce pipe, prepared by
8443/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8444/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8445/// server before spawn and authenticates its client after the child exists.
8446#[cfg(unix)]
8447type NonceHandoff = subc_os::LaunchNonceHandoff;
8448#[cfg(windows)]
8449type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8450#[cfg(not(any(unix, windows)))]
8451type NonceHandoff = std::convert::Infallible;
8452
8453/// Create the launch nonce that a new module process must present when it
8454/// registers or opens a consumer connection, including a replacement process
8455/// started alongside a still-serving module.
8456///
8457/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8458/// a separate candidate token so the still-serving incumbent and its consumers
8459/// keep their nonce. Both records are installed before the process exists, so
8460/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8461///
8462/// On Unix the nonce is delivered only through a pipe. It is written into
8463/// a pipe whose read end the child gets as descriptor 3, named by
8464/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8465/// process of the same user cannot read it with `ps eww`. That handoff is
8466/// returned rather than installed here, because installing it replaces
8467/// whatever the child has at descriptor 3 and so must be the last pre-exec
8468/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8469/// the Linux callback that writes the process into its cgroup. Only after that
8470/// does `handoff.install_last` register the descriptor-3 replacement, so it
8471/// cannot close a descriptor the cgroup callback still needs.
8472/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8473fn apply_wire_spawn_args_for_role(
8474    command: &mut Command,
8475    spec: &ModuleSpec,
8476    connection_file_path: Option<&std::path::Path>,
8477    handle: Option<&SupervisorHandle>,
8478    role: SpawnRole,
8479) -> Result<Option<NonceHandoff>, SuperviseError> {
8480    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8481    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8482    // included: a daemon started from a module's process tree inherits it,
8483    // and passing it on would point the child at a descriptor it does not
8484    // have.
8485    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8486    #[cfg(windows)]
8487    {
8488        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8489        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8490    }
8491    // Remove inherited or configured copies too: withholding must mean absent.
8492    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8493    if spec.protocol == ModuleProtocol::None {
8494        return Ok(None);
8495    }
8496    if let Some(connection_file_path) = connection_file_path {
8497        command.arg(SUBC_ARG).arg(connection_file_path);
8498    }
8499
8500    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8501    // route.open attestation. Reserved modules additionally use the same nonce
8502    // for HELLO id-squatting protection. A respawn rotates both records.
8503    let nonce = generate_launch_nonce()?;
8504    if let Some(handle) = handle {
8505        match role {
8506            SpawnRole::Plain => {
8507                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8508                if spec.reserved {
8509                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8510                }
8511            }
8512            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8513        }
8514    }
8515    #[cfg(unix)]
8516    let handoff = {
8517        let handoff =
8518            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8519                program: spec.program.clone(),
8520                source,
8521                cgroup_path: None,
8522            })?;
8523        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8524        Some(handoff)
8525    };
8526    #[cfg(windows)]
8527    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8528    #[cfg(not(any(unix, windows)))]
8529    let handoff = None;
8530    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8531    // until the daemon's per-spawn delivery records show that every running
8532    // Windows module has read its secret from the named pipe.
8533    #[cfg(not(unix))]
8534    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8535    Ok(handoff)
8536}
8537
8538fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8539    command.env_remove(CK_LOG_ENV);
8540    // The spawn role is the supervisor's to set, and only on a swap candidate
8541    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8542    // it, is what makes it absent on a plain spawn: the daemon's own
8543    // environment could carry it, and so could a spec built outside daemon
8544    // config (config refuses it as an `env` key). A module reading it on a
8545    // plain restart would pick the long swap budget and leave callers waiting.
8546    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8547    for (key, value) in &spec.env {
8548        // cortexkit-log currently exposes retention only as a Rust struct, not
8549        // environment names. These values are daemon-private sink metadata and
8550        // must never become a public child-process contract by being inherited.
8551        if matches!(
8552            key.as_str(),
8553            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8554        ) || key == SUBC_SPAWN_ROLE_ENV
8555        {
8556            continue;
8557        }
8558        command.env(key, value);
8559    }
8560}
8561
8562#[cfg(windows)]
8563fn prepare_windows_nonce_pipe(
8564    command: &mut Command,
8565    spec: &ModuleSpec,
8566    nonce: &str,
8567) -> Option<NonceHandoff> {
8568    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8569        Ok(handoff) => {
8570            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8571            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8572            // still register until every module reads the pipe. A reader without
8573            // this variable set to env returns the pipe error instead of using
8574            // an inherited or configured environment copy.
8575            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8576            Some(handoff)
8577        }
8578        Err(error) => {
8579            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8580            None
8581        }
8582    }
8583}
8584
8585#[cfg(windows)]
8586fn start_windows_nonce_pipe(
8587    handoff: Option<NonceHandoff>,
8588    pid: u32,
8589    spec: &ModuleSpec,
8590) -> Option<subc_os::LaunchNoncePipeDelivery> {
8591    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8592    // same maximum wait used for registration during a reload. Older modules
8593    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8594    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8595        Ok(delivery) => Some(delivery),
8596        Err(error) => {
8597            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8598            None
8599        }
8600    })
8601}
8602
8603fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8604    #[cfg(windows)]
8605    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8606        return Some(
8607            if snapshot
8608                .pipe_consumed
8609                .as_ref()
8610                .is_some_and(|consumed| consumed.get().is_some())
8611            {
8612                "pipe"
8613            } else {
8614                "env"
8615            }
8616            .to_owned(),
8617        );
8618    }
8619    let _ = snapshot;
8620    None
8621}
8622
8623/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8624/// of a blue/green swap.
8625#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8626enum SpawnRole {
8627    Plain,
8628    SwapCandidate,
8629}
8630
8631/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8632/// `apply_child_env` has already removed the variable for every spawn.
8633fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8634    if role == SpawnRole::SwapCandidate {
8635        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8636    }
8637}
8638
8639fn previous_instance_error(
8640    spec: &ModuleSpec,
8641    pid: u32,
8642    reason: crate::live_children::RetainedReason,
8643) -> SuperviseError {
8644    SuperviseError::Spawn {
8645        program: spec.program.clone(),
8646        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8647        cgroup_path: None,
8648    }
8649}
8650
8651fn spawn_child(
8652    spec: &ModuleSpec,
8653    connection_file_path: Option<&std::path::Path>,
8654    handle: Option<&SupervisorHandle>,
8655    ring: &Arc<Mutex<StderrRing>>,
8656    capture_logs_dir: Option<&std::path::Path>,
8657    roster: &ChildRoster,
8658    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8659) -> Result<SupervisedChild, SuperviseError> {
8660    spawn_child_in_slot(
8661        spec,
8662        connection_file_path,
8663        handle,
8664        ring,
8665        capture_logs_dir,
8666        roster,
8667        #[cfg(target_os = "linux")]
8668        cgroup_placement,
8669        SpawnRole::Plain,
8670        false,
8671    )
8672}
8673
8674/// Spawn one process of `spec` into a slot.
8675///
8676/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8677/// A swap candidate needs a different cgroup from the process it is replacing,
8678/// which is still alive: in the same cgroup the two would be one kill domain,
8679/// and killing a failed candidate could take the incumbent with it.
8680///
8681/// The stderr capture file is `<module_id>.stderr.log` for every process of
8682/// the module, whichever slot it is in, because that is the one file
8683/// `ck module logs` reads. During a swap's overlap both processes append to it;
8684/// the daemon writes whole lines, so the two interleave by line, which is also
8685/// the merged view an operator wants while a swap runs.
8686#[allow(clippy::too_many_arguments)]
8687fn spawn_child_in_slot(
8688    spec: &ModuleSpec,
8689    connection_file_path: Option<&std::path::Path>,
8690    handle: Option<&SupervisorHandle>,
8691    ring: &Arc<Mutex<StderrRing>>,
8692    capture_logs_dir: Option<&std::path::Path>,
8693    roster: &ChildRoster,
8694    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8695    role: SpawnRole,
8696    alternate_slot: bool,
8697) -> Result<SupervisedChild, SuperviseError> {
8698    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8699        return Err(previous_instance_error(spec, pid, reason));
8700    }
8701    if roster.is_closed() {
8702        return Err(SuperviseError::Spawn {
8703            program: spec.program.clone(),
8704            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8705            cgroup_path: None,
8706        });
8707    }
8708    #[cfg(target_os = "linux")]
8709    let cgroup_name = {
8710        // Slot names alone are not kill domains: a retired incumbent may still
8711        // be draining when a later enable/restart spawns into the same slot.
8712        // Decimal entropy keeps the suffix unambiguous; Placement performs
8713        // the module-id escaping and constructs the filesystem path.
8714        if cgroup_placement.is_none() {
8715            swap::cgroup_name(&spec.module_id, alternate_slot)
8716        } else {
8717            let nonce = generate_launch_nonce()?;
8718            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8719            // Leave room for byte escaping and the suffix under NAME_MAX. The
8720            // label is only for humans; the nonce identifies the kill domain.
8721            let mut end = spec.module_id.len().min(64);
8722            while !spec.module_id.is_char_boundary(end) {
8723                end -= 1;
8724            }
8725            format!(
8726                "{}_{suffix}",
8727                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8728            )
8729        }
8730    };
8731    #[cfg(not(target_os = "linux"))]
8732    let _ = alternate_slot;
8733    #[cfg(target_os = "macos")]
8734    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8735    #[cfg(windows)]
8736    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8737        program: spec.program.clone(),
8738        source,
8739        cgroup_path: None,
8740    })?;
8741    #[cfg(windows)]
8742    let mut command = Command::new(image_capture.path());
8743    #[cfg(not(any(target_os = "macos", windows)))]
8744    let mut command = Command::new(&spec.program);
8745    command.args(&spec.args);
8746    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8747    // that is the whole of the intent, so remove that one key rather than the
8748    // environment.
8749    //
8750    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8751    // and took the POSIX environment with it. Modules spawned that way had no
8752    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8753    // logging:
8754    //
8755    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8756    //     both unset it fell back to the temp dir alone and `ck` could not find
8757    //     a daemon running on the same machine from inside any module's process
8758    //     tree — reporting a path the file has never lived at, which reads as
8759    //     "the daemon did not write its file".
8760    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8761    //     the RELATIVE `.local/share`, so a module deriving its own store path
8762    //     resolved it against its own CWD. That is the store-fragmentation
8763    //     defect the daemon already refuses in config (`parse_doc` rejects a
8764    //     relative `storage.data_home`) arriving by derivation instead.
8765    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8766    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8767    //     quietly rather than erroring.
8768    //
8769    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8770    // offered one candidate under /tmp while the file sat in /run/user/1000.
8771    //
8772    // A configured module is unaffected either way: `module_spec()` puts the
8773    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8774    // wins over anything ambient.
8775    apply_child_env(&mut command, spec);
8776    apply_spawn_role(&mut command, role);
8777    let nonce_handoff =
8778        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8779
8780    #[cfg(target_os = "linux")]
8781    let cgroup_path = cgroup_placement
8782        .map(|placement| placement.module_path(&cgroup_name))
8783        .transpose()
8784        .map_err(|source| SuperviseError::Cgroup {
8785            module_id: spec.module_id.clone(),
8786            source,
8787        })?;
8788    #[cfg(not(target_os = "linux"))]
8789    let cgroup_path: Option<PathBuf> = None;
8790    #[cfg(target_os = "linux")]
8791    if let Some(path) = &cgroup_path {
8792        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8793            if let Some(placement) = cgroup_placement {
8794                remove_module_cgroup(placement, &cgroup_name);
8795            }
8796            return Err(error);
8797        }
8798    }
8799
8800    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8801        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8802        match ChildOutputSink::open(&path, capture_retention(spec)) {
8803            Ok(sink) => sink,
8804            Err(error) => {
8805                warn!(
8806                    module_id = %spec.module_id,
8807                    path = %path.display(),
8808                    error = %error,
8809                    "could not open child output capture file; forwarding to stderr"
8810                );
8811                ChildOutputSink::Stderr
8812            }
8813        }
8814    } else {
8815        ChildOutputSink::Stderr
8816    };
8817
8818    command.stdout(Stdio::piped());
8819    command.stderr(Stdio::piped());
8820    command.kill_on_drop(true);
8821    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8822    // before exec). In the daemon's group, a service manager that kills the
8823    // job's process group when the daemon exits (launchd's default) killed
8824    // every module at the same moment its control connection closed, so no
8825    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8826    // module is reached only by the daemon: the EOF it sees when its
8827    // connection closes, and the bounded stop in `child_roster` for anything
8828    // still running after that. On Linux this composes with the cgroup
8829    // placement above: that is a pre_exec write to cgroup.procs, std performs
8830    // setpgid in the child before running pre_exec callbacks, and the two
8831    // change independent process attributes.
8832    //
8833    // stdin is /dev/null because a process outside the terminal's foreground
8834    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8835    // by hand would otherwise hand down. Under a service manager stdin is
8836    // already /dev/null.
8837    #[cfg(unix)]
8838    command.process_group(0);
8839    command.stdin(Stdio::null());
8840    // The LAST pre-exec step, after the cgroup placement above: installing the
8841    // nonce at descriptor 3 replaces whatever the child had there, which could
8842    // be the descriptor an earlier step writes through.
8843    #[cfg(unix)]
8844    if let Some(handoff) = nonce_handoff {
8845        handoff.install_last(command.as_std_mut());
8846    }
8847    #[cfg(not(any(unix, windows)))]
8848    let _ = nonce_handoff;
8849
8850    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8851    // cannot run a single instruction -- and therefore cannot spawn a
8852    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8853    // other two steps and why the window matters.
8854    #[cfg(windows)]
8855    {
8856        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8857            program: spec.program.clone(),
8858            source,
8859            cgroup_path: None,
8860        })?;
8861        subc_jobobject::suspend_on_create_async(&mut command);
8862    }
8863    // Timestamp before the spawn call: on Unix the child can serve before that
8864    // call even returns. This conservative bound never labels an older child
8865    // as a replacement for a request received during process creation.
8866    let spawned_at = Instant::now();
8867    let mut child = match command.spawn() {
8868        Ok(child) => child,
8869        Err(source) => {
8870            #[cfg(target_os = "linux")]
8871            if let Some(placement) = cgroup_placement {
8872                remove_module_cgroup(placement, &cgroup_name);
8873            }
8874            return Err(SuperviseError::Spawn {
8875                program: spec.program.clone(),
8876                source,
8877                cgroup_path,
8878            });
8879        }
8880    };
8881    #[cfg(test)]
8882    spawn_record_tests::after_create(&child, spec);
8883    // The parent must close its writer now: the acknowledgement pipe reports EOF
8884    // only when every writer is gone, and the child's copy closes when the
8885    // trampoline replaces itself with the module. Command holds only an integer
8886    // in its pre_exec callback, not another writer.
8887    #[cfg(target_os = "macos")]
8888    drop(exec_ack);
8889
8890    // Bind while the child is still suspended, before job assignment resumes it.
8891    // Keeping the executable handle open across CreateProcessW denies rename
8892    // and deletion, so the file cannot be exchanged during spawn. Bind that
8893    // file ID to the process handle returned by spawn, never a reopened PID.
8894    #[cfg(windows)]
8895    let windows_image = match image_capture.bind_tokio(&child) {
8896        Ok(image) => Arc::new(image),
8897        Err(source) => {
8898            kill_and_reap_rejected_child(&mut child);
8899            return Err(SuperviseError::Spawn {
8900                program: spec.program.clone(),
8901                source,
8902                cgroup_path: None,
8903            });
8904        }
8905    };
8906    // Assign the suspended child to its module job; resume only after its
8907    // process identity has been durably recorded for orphan recovery.
8908    #[cfg(windows)]
8909    let job = contain_spawned_child(&mut child, spec)?;
8910    let spawned_at_ms = unix_ms_now();
8911    #[cfg(not(windows))]
8912    let spawned_from = spec.program.clone();
8913    #[cfg(windows)]
8914    let spawned_from = windows_image.path().to_path_buf();
8915    #[cfg(not(windows))]
8916    let spawned_file_identity = spawned_file_identity(&spawned_from);
8917    #[cfg(windows)]
8918    let spawned_file_identity = Some(SpawnedFileIdentity {
8919        device: 0,
8920        inode: 0,
8921        windows_image: Some(Arc::clone(&windows_image)),
8922    });
8923    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8924        program: spec.program.clone(),
8925        source: io::Error::other("spawned child exposed no live pid"),
8926        cgroup_path: cgroup_path.clone(),
8927    })?;
8928    #[cfg(windows)]
8929    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8930    #[cfg(not(windows))]
8931    let process_start_time = crate::provenance::process_start_time(pid);
8932    #[cfg(windows)]
8933    let process_start_time = Some(windows_image.start_time());
8934    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8935    #[cfg(all(test, target_os = "macos"))]
8936    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8937    // Record the committed kernel image, not the configured pathname: PATH
8938    // lookup and shebang interpretation can select a different executable.
8939    // Linux's posix_spawn can return before that image is installed, so the
8940    // observer also waits for the kernel's first-exec flag to clear.
8941    #[cfg(not(windows))]
8942    let recorded_image = observe_spawned_image(pid);
8943    // spawn() confirms only the first exec, into the trampoline. Never persist
8944    // the trampoline image; the asynchronous acknowledgement publishes the
8945    // module image once the trampoline has replaced itself with the module.
8946    #[cfg(target_os = "macos")]
8947    let recorded_image = if privacy_exec.is_some() {
8948        None
8949    } else {
8950        recorded_image
8951    };
8952    #[cfg(target_os = "linux")]
8953    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8954    #[cfg(not(target_os = "linux"))]
8955    let recorded_cgroup_name = None;
8956    let roster_guard = roster.admit(
8957        spec.module_id.clone(),
8958        pid,
8959        spec.protocol,
8960        process_start_time,
8961        crate::child_roster::RecordedIdentity {
8962            #[cfg(not(windows))]
8963            start_time: recorded_image.map(|image| image.start_time),
8964            #[cfg(windows)]
8965            start_time: process_start_time,
8966            #[cfg(not(windows))]
8967            executable: recorded_image
8968                .and_then(|image| image.executable)
8969                .map(crate::live_children::ExecutableIdentity::from),
8970            #[cfg(windows)]
8971            executable: Some(crate::live_children::ExecutableIdentity::from(
8972                windows_image.identity(),
8973            )),
8974            cgroup_name: recorded_cgroup_name,
8975            #[cfg(target_os = "linux")]
8976            cgroup_placement: cgroup_placement.cloned(),
8977        },
8978    );
8979    let roster_guard = match roster_guard {
8980        Ok(guard) => guard,
8981        Err(failure) => {
8982            return failure.with_cleanup(|source| {
8983                #[cfg(windows)]
8984                if let Some(job) = &job {
8985                    let _ = job.terminate();
8986                }
8987                #[cfg(target_os = "linux")]
8988                kill_module_cgroup(cgroup_placement, &cgroup_name);
8989                kill_and_reap_rejected_child(&mut child);
8990                #[cfg(target_os = "linux")]
8991                if let Some(placement) = cgroup_placement {
8992                    remove_module_cgroup(placement, &cgroup_name);
8993                }
8994                Err(SuperviseError::Spawn {
8995                    program: spec.program.clone(),
8996                    source: io::Error::other(format!(
8997                        "could not durably record module child: {source}"
8998                    )),
8999                    cgroup_path,
9000                })
9001            });
9002        }
9003    };
9004    // The pre-spawn check can pass just before shutdown begins. Ownership is
9005    // published at the start of admission, even while a record write is queued.
9006    // If shutdown already saw an empty roster before that publication, this
9007    // check sees the roster closed and ends the child before starting a module.
9008    if roster.is_closed() {
9009        // This child was never admitted, so there is no module protocol shutdown to wait for.
9010        #[cfg(target_os = "linux")]
9011        kill_module_cgroup(cgroup_placement, &cgroup_name);
9012        if let Err(error) = child.start_kill() {
9013            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9014        }
9015        #[cfg(target_os = "linux")]
9016        if let Some(placement) = cgroup_placement {
9017            // This spawn was never admitted, so shutdown has no roster entry
9018            // to await. Do not detach its cleanup: the runtime could exit
9019            // before that task reaps the rejected child and removes its group.
9020            while matches!(child.try_wait(), Ok(None)) {
9021                std::thread::yield_now();
9022            }
9023            if matches!(
9024                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9025                subc_cgroup::KillOutcome::Killed
9026            ) {
9027                if let Ok(path) = placement.module_path(&cgroup_name) {
9028                    while std::fs::read_to_string(path.join("cgroup.events"))
9029                        .ok()
9030                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9031                    {
9032                        std::thread::yield_now();
9033                    }
9034                }
9035            }
9036            remove_module_cgroup(placement, &cgroup_name);
9037        }
9038        drop(roster_guard);
9039        return Err(SuperviseError::Spawn {
9040            program: spec.program.clone(),
9041            source: io::Error::other(
9042                "the daemon began shutting down while this process was starting; ended it",
9043            ),
9044            cgroup_path,
9045        });
9046    }
9047
9048    #[cfg(windows)]
9049    if let Err(error) = resume_suspended_child(pid, spec) {
9050        kill_and_reap_rejected_child(&mut child);
9051        return Err(error);
9052    }
9053
9054    let stdout_pump = match child.stdout.take() {
9055        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9056        None => {
9057            warn!(
9058                module_id = %spec.module_id,
9059                "spawned child exposed no stdout pipe; file capture will be incomplete"
9060            );
9061            None
9062        }
9063    };
9064    let stderr_pump = match child.stderr.take() {
9065        Some(stderr) => {
9066            let generation = ring
9067                .lock()
9068                .unwrap_or_else(|poisoned| poisoned.into_inner())
9069                .begin_process();
9070            Some(StderrPump {
9071                task: tokio::spawn(pump_stderr_to(
9072                    stderr,
9073                    Arc::clone(ring),
9074                    generation,
9075                    output_sink,
9076                )),
9077                generation,
9078            })
9079        }
9080        None => {
9081            // Spawning succeeded but the pipe did not materialise. Recording it as
9082            // uncaptured keeps the tail honest: the alternative is an empty tail
9083            // that reads as a module which printed nothing.
9084            ring.lock()
9085                .unwrap_or_else(|poisoned| poisoned.into_inner())
9086                .mark_not_captured("stderr pipe was not available on spawn");
9087            warn!(
9088                module_id = %spec.module_id,
9089                "spawned child exposed no stderr pipe; tail will be unavailable"
9090            );
9091            None
9092        }
9093    };
9094
9095    let exec = Arc::new(crate::activity::ExecOutcome::new());
9096    #[cfg(target_os = "macos")]
9097    let pending_exec = privacy_exec.is_some();
9098    #[cfg(not(target_os = "macos"))]
9099    let pending_exec = false;
9100    if !pending_exec {
9101        exec.finish(crate::activity::ExecState::CommittedExec);
9102    }
9103    Ok(SupervisedChild {
9104        child,
9105        exec: crate::activity::ExecOwner(exec),
9106        #[cfg(windows)]
9107        nonce_delivery,
9108        protocol: spec.protocol,
9109        #[cfg(target_os = "linux")]
9110        module_id: cgroup_name,
9111        #[cfg(target_os = "linux")]
9112        cgroup_placement: cgroup_placement.cloned(),
9113        #[cfg(windows)]
9114        job,
9115        stdout_pump,
9116        stderr_pump,
9117        stderr_ring: Arc::clone(ring),
9118        spawned_at,
9119        spawned_at_ms,
9120        spawned_from,
9121        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9122        spawned_file_identity,
9123        process_start_time,
9124        process_identity,
9125        pid,
9126        roster_guard: Some(roster_guard),
9127        #[cfg(target_os = "macos")]
9128        privacy_exec,
9129        #[cfg(target_os = "macos")]
9130        report_ready: Arc::new(OnceLock::new()),
9131        spawn_failure: None,
9132    })
9133}
9134
9135#[cfg(test)]
9136mod spawn_record_tests {
9137    use super::*;
9138
9139    thread_local! {
9140        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9141    }
9142
9143    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9144        LAST_PID.set(child.id());
9145        #[cfg(windows)]
9146        if let Some((_, address)) = _spec
9147            .env
9148            .iter()
9149            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9150        {
9151            use std::io::{Read, Write};
9152            let mut stream = std::net::TcpStream::connect(address).unwrap();
9153            stream
9154                .write_all(&child.id().unwrap().to_ne_bytes())
9155                .unwrap();
9156            let mut release = [0];
9157            stream.read_exact(&mut release).unwrap();
9158        }
9159    }
9160
9161    #[cfg(windows)]
9162    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9163        if spec
9164            .env
9165            .iter()
9166            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9167        {
9168            Err(io::Error::other("injected module job assignment failure"))
9169        } else {
9170            Ok(())
9171        }
9172    }
9173
9174    fn spec() -> ModuleSpec {
9175        ModuleSpec {
9176            module_id: "spawn-record-worker".into(),
9177            #[cfg(unix)]
9178            program: "/bin/sleep".into(),
9179            #[cfg(windows)]
9180            program: std::env::current_exe().unwrap(),
9181            #[cfg(unix)]
9182            args: vec!["60".into()],
9183            #[cfg(windows)]
9184            args: vec![
9185                "--exact".into(),
9186                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9187                "--nocapture".into(),
9188            ],
9189            env: vec![],
9190            reserved: false,
9191            reserved_prefixes: vec![],
9192            protocol: ModuleProtocol::None,
9193            overlap: Default::default(),
9194        }
9195    }
9196
9197    #[tokio::test]
9198    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9199        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9200        let supervisor =
9201            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9202                .with_live_children_record(home.join("live-children.json"));
9203        crate::live_children::fail_next_file_sync();
9204        let result = supervisor.spawn(spec());
9205        let failed = result.is_err();
9206        if let Ok(module) = result {
9207            module.stop().await.unwrap();
9208        }
9209        assert!(failed, "record failure must fail spawn");
9210        let pid = LAST_PID.get().expect("the real OS child was created");
9211        assert!(
9212            subc_os::Process::open(pid).unwrap().is_none(),
9213            "child {pid} survived rejected admission or was not reaped"
9214        );
9215    }
9216
9217    #[cfg(target_os = "macos")]
9218    #[test]
9219    #[ignore = "manual on-device latency measurement"]
9220    fn durable_record_write_latency_50() {
9221        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9222        let path = home.join("live-children.json");
9223        let children = vec![crate::live_children::LiveChild {
9224            module_id: "latency-worker".into(),
9225            pid: std::process::id(),
9226            protocol: ModuleProtocol::None,
9227            start_time: Some(1),
9228            executable: None,
9229            cgroup_name: None,
9230        }];
9231        let mut samples = Vec::new();
9232        for _ in 0..50 {
9233            let start = std::time::Instant::now();
9234            crate::live_children::write_record(&path, &children).unwrap();
9235            samples.push(start.elapsed());
9236        }
9237        samples.sort();
9238        eprintln!(
9239            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9240            (samples[24] + samples[25]) / 2,
9241            samples[47]
9242        );
9243    }
9244
9245    #[cfg(windows)]
9246    #[test]
9247    fn windows_store_touch_fixture() {
9248        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9249            std::fs::write(path, b"module ran").unwrap();
9250        }
9251    }
9252
9253    #[cfg(windows)]
9254    #[tokio::test]
9255    async fn windows_daemon_fixture() {
9256        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9257            return;
9258        };
9259        let mut spec = spec();
9260        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9261        let supervisor =
9262            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9263        let _module = supervisor.spawn(spec).unwrap();
9264    }
9265
9266    #[cfg(windows)]
9267    #[test]
9268    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9269        use std::io::Read;
9270        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9271        // An outer job models a scheduler/service host. The daemon's root and
9272        // root job must nest without requiring breakaway from this job.
9273        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9274        command
9275            .args([
9276                "--exact",
9277                "supervise::spawn_record_tests::windows_daemon_fixture",
9278                "--nocapture",
9279            ])
9280            .env(
9281                "SUBC_TEST_DAEMON_BARRIER",
9282                listener.local_addr().unwrap().to_string(),
9283            );
9284        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9285        listener.set_nonblocking(true).unwrap();
9286        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9287        let mut stream = loop {
9288            if let Ok((stream, _)) = listener.accept() {
9289                break stream;
9290            }
9291            assert!(
9292                daemon.child.try_wait().unwrap().is_none(),
9293                "daemon failed before rendezvous"
9294            );
9295            assert!(
9296                std::time::Instant::now() < deadline,
9297                "daemon did not reach CreateProcess failpoint"
9298            );
9299            std::thread::yield_now();
9300        };
9301        stream
9302            .set_read_timeout(Some(Duration::from_secs(30)))
9303            .unwrap();
9304        let mut pid = [0; 4];
9305        stream.read_exact(&mut pid).unwrap();
9306        let pid = u32::from_ne_bytes(pid);
9307        assert!(subc_jobobject::process_exists(pid));
9308        // Terminate only the daemon, not its outer job: otherwise the host job
9309        // would also kill a child whose daemon never joined its root job.
9310        daemon.child.kill().unwrap();
9311        daemon.child.wait().unwrap();
9312        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9313        if !exited {
9314            daemon.job.terminate().unwrap();
9315        }
9316        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9317    }
9318
9319    #[cfg(windows)]
9320    #[tokio::test]
9321    async fn windows_containment_failure_never_resumes() {
9322        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9323        let sentinel = home.join("store-touched");
9324        let mut spec = spec();
9325        spec.env
9326            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9327        spec.env.push((
9328            "SUBC_TEST_STORE_TOUCH".into(),
9329            sentinel.display().to_string(),
9330        ));
9331        let supervisor =
9332            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9333        let result = supervisor.spawn(spec);
9334        // A version that resumes despite assignment failure must have time to
9335        // execute its sentinel before dropping the returned module kills it.
9336        if let Ok(module) = result {
9337            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9338                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9339                    tokio::task::yield_now().await;
9340                }
9341            })
9342            .await;
9343            let _ = module.stop().await;
9344            panic!(
9345                "assignment failure resumed a module; store touched={}",
9346                sentinel.exists()
9347            );
9348        }
9349        assert!(
9350            !sentinel.exists(),
9351            "refused module executed store-touch sentinel"
9352        );
9353        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9354    }
9355
9356    #[cfg(windows)]
9357    #[tokio::test]
9358    async fn windows_nested_daemon_fixture() {
9359        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9360            return;
9361        };
9362        let mut spec = spec();
9363        spec.env.push((
9364            "SUBC_TEST_STORE_TOUCH".into(),
9365            sentinel.to_str().unwrap().into(),
9366        ));
9367        let supervisor =
9368            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9369        let module = supervisor
9370            .spawn(spec)
9371            .expect("per-module job must nest below daemon and host jobs");
9372        tokio::time::timeout(Duration::from_secs(10), async {
9373            while !PathBuf::from(&sentinel).exists() {
9374                tokio::task::yield_now().await;
9375            }
9376        })
9377        .await
9378        .unwrap();
9379        module.stop().await.unwrap();
9380    }
9381
9382    #[cfg(windows)]
9383    #[test]
9384    fn windows_module_job_nests_under_host_job() {
9385        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9386        let sentinel = home.join("nested-store-touched");
9387        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9388        command
9389            .args([
9390                "--exact",
9391                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9392                "--nocapture",
9393            ])
9394            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9395        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9396        assert!(daemon.child.wait().unwrap().success());
9397        assert!(sentinel.exists(), "nested protocol:none module never ran");
9398    }
9399}
9400
9401#[cfg(target_os = "linux")]
9402pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9403    use subc_cgroup::KillOutcome;
9404    match subc_cgroup::kill_module(placement, module_id) {
9405        KillOutcome::Killed => {}
9406        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9407            debug!(
9408                module_id,
9409                "cgroup tree kill unavailable; using direct-child kill"
9410            );
9411        }
9412        KillOutcome::IoError { path, error } => {
9413            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9414        }
9415    }
9416}
9417
9418/// Assign a freshly spawned Windows child without ever resuming on failure.
9419///
9420/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9421/// child assigned **while it is still suspended** (step 1 is
9422/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9423/// the child's roster admission is durable. Failed containment kills and reaps
9424/// the suspended child instead of running an unprotected module.
9425#[cfg(windows)]
9426fn contain_spawned_child(
9427    child: &mut Child,
9428    spec: &ModuleSpec,
9429) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9430    let assignment = || {
9431        let job = subc_jobobject::JobObject::new()?;
9432        #[cfg(test)]
9433        spawn_record_tests::assignment_failpoint(spec)?;
9434        job.assign(child)?;
9435        Ok::<_, io::Error>(job)
9436    };
9437    match assignment() {
9438        Ok(job) => Ok(Some(job)),
9439        Err(source) => {
9440            kill_and_reap_rejected_child(child);
9441            Err(SuperviseError::Spawn {
9442                program: spec.program.clone(),
9443                source: io::Error::other(format!(
9444                    "Windows module job containment failed: {source}"
9445                )),
9446                cgroup_path: None,
9447            })
9448        }
9449    }
9450}
9451
9452/// Rejection is synchronous: do not return a spawn error while the child can
9453/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9454fn kill_and_reap_rejected_child(child: &mut Child) {
9455    #[cfg(unix)]
9456    if let Some(pid) = child
9457        .id()
9458        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9459    {
9460        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9461    }
9462    let _ = child.start_kill();
9463    while matches!(child.try_wait(), Ok(None)) {
9464        std::thread::yield_now();
9465    }
9466}
9467
9468/// Resume a suspended child, killing it if it cannot be started.
9469///
9470/// A suspended process holds a pid and does nothing, so there is no useful
9471/// state to return: the caller gets an error and the spawn fails.
9472#[cfg(windows)]
9473fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9474    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9475        // Kill it here rather than leaving a suspended process for the caller
9476        // to notice; `kill_on_drop` would eventually do this, but the module
9477        // would have been reported as running in between.
9478        let _ = std::process::Command::new("taskkill.exe")
9479            .args(["/PID", &pid.to_string(), "/T", "/F"])
9480            .stdin(Stdio::null())
9481            .stdout(Stdio::null())
9482            .stderr(Stdio::null())
9483            .status();
9484        return Err(SuperviseError::Spawn {
9485            program: spec.program.clone(),
9486            source,
9487            cgroup_path: None,
9488        });
9489    }
9490    Ok(())
9491}
9492
9493#[cfg(target_os = "linux")]
9494fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9495    match placement.remove_module(module_id) {
9496        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9497        Err(error) => warn!(
9498            module_id,
9499            error = %error,
9500            "could not remove module cgroup after process exit; continuing teardown"
9501        ),
9502    }
9503}
9504
9505#[cfg(target_os = "linux")]
9506async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9507    // Reaping the direct child is not proof its descendants exited. End the
9508    // residual tree and wait for the kernel's population fact before rmdir;
9509    // otherwise a successful parent wait leaks a directory on each restart.
9510    if matches!(
9511        subc_cgroup::kill_module(Some(placement), module_id),
9512        subc_cgroup::KillOutcome::Killed
9513    ) {
9514        if let Ok(path) = placement.module_path(module_id) {
9515            while std::fs::read_to_string(path.join("cgroup.events"))
9516                .ok()
9517                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9518            {
9519                sleep(Duration::from_millis(1)).await;
9520            }
9521        }
9522    }
9523    remove_module_cgroup(placement, module_id);
9524}
9525
9526#[cfg(target_os = "linux")]
9527fn apply_cgroup_placement(
9528    command: &mut Command,
9529    spec: &ModuleSpec,
9530    path: &std::path::Path,
9531) -> Result<(), SuperviseError> {
9532    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9533        module_id: spec.module_id.clone(),
9534        source,
9535    })
9536}
9537
9538fn capture_retention(spec: &ModuleSpec) -> Retention {
9539    let defaults = Retention::default();
9540    let value = |name: &str| {
9541        spec.env
9542            .iter()
9543            .rev()
9544            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9545    };
9546    Retention {
9547        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9548            .and_then(|value| value.parse().ok())
9549            .unwrap_or(defaults.max_file_mb),
9550        keep: value(CAPTURE_KEEP_ENV)
9551            .and_then(|value| value.parse().ok())
9552            .unwrap_or(defaults.keep),
9553        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9554            .and_then(|value| value.parse().ok())
9555            .unwrap_or(defaults.max_age_days),
9556    }
9557}
9558
9559/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9560/// module's registration to the exact process the supervisor spawned.
9561fn generate_launch_nonce() -> Result<String, SuperviseError> {
9562    let mut bytes = [0u8; 32];
9563    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9564        reason: source.to_string(),
9565    })?;
9566    let mut hex = String::with_capacity(64);
9567    for b in bytes {
9568        use std::fmt::Write;
9569        let _ = write!(hex, "{b:02x}");
9570    }
9571    Ok(hex)
9572}
9573
9574/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9575/// signal about how many leading bytes matched.
9576fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9577    if a.len() != b.len() {
9578        return false;
9579    }
9580    let mut diff = 0u8;
9581    for (x, y) in a.iter().zip(b.iter()) {
9582        diff |= x ^ y;
9583    }
9584    diff == 0
9585}
9586
9587/// The kernel's image after an acknowledged exec, shared by ordinary launches
9588/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9589/// not identities inferred from a configured pathname.
9590#[cfg(not(windows))]
9591fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9592    let process = subc_os::Process::open(pid).ok().flatten()?;
9593    #[cfg(target_os = "linux")]
9594    return sample_after_linux_exec(
9595        || {
9596            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9597            let committed = linux_exec_committed(&stat);
9598            #[cfg(test)]
9599            spawn_image_tests::report_exec_state(committed);
9600            committed
9601        },
9602        || process.observe(),
9603    );
9604    #[cfg(not(target_os = "linux"))]
9605    process.observe()
9606}
9607
9608/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9609/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9610/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9611#[cfg(target_os = "linux")]
9612fn linux_exec_committed(stat: &str) -> Option<bool> {
9613    const PF_FORKNOEXEC: u64 = 0x40;
9614    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9615    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9616}
9617
9618#[cfg(target_os = "linux")]
9619fn sample_after_linux_exec(
9620    mut exec_committed: impl FnMut() -> Option<bool>,
9621    observe: impl FnOnce() -> Option<subc_os::Observation>,
9622) -> Option<subc_os::Observation> {
9623    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9624    while !exec_committed()? {
9625        // A failed or indefinitely stalled exec must not publish the inherited
9626        // image as authority to signal a future orphan. Missing identity keeps
9627        // the existing conservative orphan-sweep behavior.
9628        if std::time::Instant::now() >= deadline {
9629            return None;
9630        }
9631        std::thread::sleep(Duration::from_micros(50));
9632    }
9633    observe()
9634}
9635
9636#[cfg(all(test, target_os = "linux"))]
9637mod spawn_image_tests {
9638    use super::*;
9639    use std::cell::{Cell, RefCell};
9640
9641    enum ExecEvent {
9642        Uncommitted,
9643        Image(Option<subc_os::Observation>),
9644    }
9645
9646    thread_local! {
9647        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9648    }
9649
9650    pub(super) fn report_exec_state(committed: Option<bool>) {
9651        if committed == Some(false) {
9652            EXEC_EVENTS.with(|events| {
9653                if let Some(events) = events.borrow_mut().take() {
9654                    let _ = events.send(ExecEvent::Uncommitted);
9655                }
9656            });
9657        }
9658    }
9659
9660    #[test]
9661    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9662        use std::io::{Read, Write};
9663        use std::os::fd::AsRawFd;
9664
9665        let descriptor = std::fs::File::open("/dev/null").unwrap();
9666        let mut command = std::process::Command::new("/bin/sleep");
9667        command.arg("30");
9668        let (mut ready, mut release) =
9669            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9670                .unwrap();
9671        let spawn = std::thread::spawn(move || command.spawn());
9672        let mut message = [0u8; 8];
9673        ready.read_exact(&mut message).unwrap();
9674        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9675        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9676        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9677        let (events, received) = std::sync::mpsc::channel();
9678        let observer = std::thread::spawn(move || {
9679            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9680            let image = observe_spawned_image(pid);
9681            let _ = events.send(ExecEvent::Image(image));
9682        });
9683        // Release only once the production observer has seen the uncommitted
9684        // exec, or has incorrectly returned the inherited image. Either way,
9685        // release and reap before asserting so a regression cannot leak a child.
9686        let first = received.recv_timeout(Duration::from_secs(5));
9687        release.write_all(&[1]).unwrap();
9688        let mut child = spawn.join().unwrap().unwrap();
9689        let after = match first {
9690            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9691                Ok(ExecEvent::Image(image)) => image,
9692                _ => None,
9693            },
9694            Ok(ExecEvent::Image(image)) => image,
9695            Err(_) => None,
9696        };
9697        observer.join().unwrap();
9698        child.kill().unwrap();
9699        child.wait().unwrap();
9700
9701        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9702        assert_eq!(
9703            inherited.unwrap().executable,
9704            subc_os::file_identity(&std::env::current_exe().unwrap())
9705        );
9706        assert_eq!(
9707            after.unwrap().executable,
9708            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9709        );
9710    }
9711
9712    #[test]
9713    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9714        let committed = Cell::new(false);
9715        let inherited = subc_os::Observation {
9716            start_time: 23,
9717            executable: Some(subc_os::FileIdentity {
9718                device: 29,
9719                inode: 31,
9720            }),
9721        };
9722        let module = subc_os::Observation {
9723            start_time: 23,
9724            executable: Some(subc_os::FileIdentity {
9725                device: 29,
9726                inode: 37,
9727            }),
9728        };
9729        let polls = Cell::new(0);
9730        let image = sample_after_linux_exec(
9731            || {
9732                polls.set(polls.get() + 1);
9733                // Hold the inherited image through the first poll, as in the
9734                // posix_spawn parent-wake window, then commit the module image.
9735                committed.set(polls.get() == 2);
9736                Some(committed.get())
9737            },
9738            || Some(if committed.get() { module } else { inherited }),
9739        );
9740        assert_eq!(image, Some(module));
9741        assert_eq!(polls.get(), 2);
9742    }
9743
9744    #[test]
9745    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9746        assert_eq!(
9747            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9748            Some(false)
9749        );
9750        assert_eq!(
9751            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9752            Some(true)
9753        );
9754        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9755        assert_eq!(linux_exec_committed("17 (a) R"), None);
9756    }
9757
9758    #[test]
9759    fn unreadable_exec_state_never_samples_an_inherited_image() {
9760        assert_eq!(
9761            sample_after_linux_exec(
9762                || None,
9763                || panic!("must not sample before exec confirmation")
9764            ),
9765            None
9766        );
9767    }
9768
9769    #[test]
9770    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9771        assert_eq!(
9772            sample_after_linux_exec(
9773                || Some(false),
9774                || panic!("must not sample after an exec timeout")
9775            ),
9776            None
9777        );
9778    }
9779}
9780
9781/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9782/// different executable after the no-delete capture was taken.
9783#[cfg(windows)]
9784fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9785    let mut program = spec.program.clone();
9786    if program.is_absolute() || program.components().count() > 1 {
9787        // An explicit path can name a PE image with no extension. Try that
9788        // exact file first, as Command does, before falling back to .exe.
9789        match subc_os::ExecutableCapture::open(&program) {
9790            Ok(capture) => return Ok(capture),
9791            Err(error)
9792                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9793            Err(error) => return Err(error),
9794        }
9795        program.set_extension("exe");
9796        return subc_os::ExecutableCapture::open(&program);
9797    }
9798    if program.extension().is_none() {
9799        program.set_extension("exe");
9800    }
9801    let custom_path = spec
9802        .env
9803        .iter()
9804        .rev()
9805        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9806    let mut directories = Vec::new();
9807    // These are Windows' default executable search locations. A configured
9808    // PATH still overrides the inherited PATH, not the system directories.
9809    if let Some(parent) = std::env::current_exe()?.parent() {
9810        directories.push(parent.to_path_buf());
9811    }
9812    directories.push(std::env::current_dir()?);
9813    if let Some(windows) = std::env::var_os("SystemRoot") {
9814        let windows = PathBuf::from(windows);
9815        directories.push(windows.join("System32"));
9816        directories.push(windows.clone());
9817    }
9818    let path = custom_path
9819        .map(|(_, value)| std::ffi::OsString::from(value))
9820        .or_else(|| std::env::var_os("PATH"));
9821    if let Some(path) = path {
9822        directories.extend(std::env::split_paths(&path));
9823    }
9824    for directory in directories {
9825        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9826            Ok(capture) => return Ok(capture),
9827            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9828            Err(error) => return Err(error),
9829        }
9830    }
9831    Err(io::Error::new(
9832        io::ErrorKind::NotFound,
9833        "resolved executable not found",
9834    ))
9835}
9836
9837fn spawn_and_mark_running(
9838    spec: &ModuleSpec,
9839    runtime: &SupervisorRuntimeConfig,
9840    snapshot: &SharedSnapshot,
9841) -> Result<SupervisedChild, SuperviseError> {
9842    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9843    let child = spawn_child(
9844        spec,
9845        runtime.connection_file_path.as_deref(),
9846        runtime.supervisor_handle.as_ref(),
9847        &runtime.stderr_ring,
9848        runtime.capture_logs_dir.as_deref(),
9849        &runtime.child_roster,
9850        #[cfg(target_os = "linux")]
9851        runtime.cgroup_placement.as_ref(),
9852    )?;
9853    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9854    if let Some(spawn) = activity_spawn {
9855        spawn.complete(snapshot);
9856    }
9857    Ok(child)
9858}
9859
9860enum RegistrationWaitOutcome {
9861    Registered,
9862    Exited(ExitReport),
9863    TimedOut,
9864}
9865
9866struct ReloadRegistrationFailure {
9867    exit_report: ExitReport,
9868    reason: String,
9869}
9870
9871#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9872enum BusyGaugeObservation {
9873    Quiescent,
9874    Busy,
9875    Omitted,
9876}
9877
9878/// What one health answer said about one declared busy gauge.
9879#[derive(Debug, Clone, PartialEq, Eq)]
9880enum GaugeVerdict {
9881    /// The gauge read 0: this gauge is not what holds the drain.
9882    Zero,
9883    /// The gauge read a positive count.
9884    NonZero(u64),
9885    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9886    /// a negative number). It counts as busy, because the daemon cannot tell
9887    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9888    NotACount(String),
9889    /// The health answer had no metrics object, or one without this gauge.
9890    Missing,
9891    /// The probe produced no health answer to read; carries the probe error's
9892    /// evidence label (`no-answer` for a probe that timed out).
9893    ProbeFailed(&'static str),
9894}
9895
9896impl GaugeVerdict {
9897    fn reads_quiet(&self) -> bool {
9898        matches!(self, Self::Zero)
9899    }
9900}
9901
9902impl fmt::Display for GaugeVerdict {
9903    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9904        match self {
9905            Self::Zero => f.write_str("zero"),
9906            Self::NonZero(value) => write!(f, "non_zero({value})"),
9907            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9908            Self::Missing => f.write_str("missing_from_health_body"),
9909            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9910        }
9911    }
9912}
9913
9914/// The longest JSON text kept for a gauge value that is not a count, so a
9915/// module reporting a large object as a gauge cannot flood the drain log.
9916const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9917
9918/// Read each declared gauge out of one health answer's metrics.
9919fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9920    let metrics = metrics.and_then(Value::as_object);
9921    gauges
9922        .iter()
9923        .map(
9924            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9925                None => GaugeVerdict::Missing,
9926                Some(value) => match value.as_u64() {
9927                    Some(0) => GaugeVerdict::Zero,
9928                    Some(count) => GaugeVerdict::NonZero(count),
9929                    None => {
9930                        let mut text = value.to_string();
9931                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9932                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9933                            while !text.is_char_boundary(end) {
9934                                end -= 1;
9935                            }
9936                            text.truncate(end);
9937                            text.push_str("...");
9938                        }
9939                        GaugeVerdict::NotACount(text)
9940                    }
9941                },
9942            },
9943        )
9944        .collect()
9945}
9946
9947/// The drain's overall reading of one health answer: omitted when any
9948/// declared gauge is missing, quiescent only when every gauge reads 0.
9949fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9950    if verdicts
9951        .iter()
9952        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9953    {
9954        BusyGaugeObservation::Omitted
9955    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9956        BusyGaugeObservation::Quiescent
9957    } else {
9958        BusyGaugeObservation::Busy
9959    }
9960}
9961
9962/// One declared gauge's history over a drain: the latest verdict on it, and
9963/// the latest value actually read for it (which a later failed probe or
9964/// missing gauge does not erase). Times are offsets from the start of the
9965/// drain's wait.
9966#[derive(Debug, Clone, Default, PartialEq, Eq)]
9967struct GaugeTrace {
9968    latest: Option<(GaugeVerdict, Duration)>,
9969    last_value: Option<(GaugeVerdict, Duration)>,
9970}
9971
9972/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9973/// its ceiling can say in its timeout line which declared gauge held it and
9974/// why. Without it, a gauge that read busy, a gauge the module never
9975/// reported, and a probe that never got an answer all looked the same: a
9976/// drain that ended at its ceiling with nothing in flight.
9977#[derive(Debug, Clone, Default, PartialEq, Eq)]
9978struct DrainGaugeTrace {
9979    /// Health probes the drain sent.
9980    probes: u32,
9981    /// Each declared gauge, in declaration order.
9982    gauges: Vec<(String, GaugeTrace)>,
9983}
9984
9985impl DrainGaugeTrace {
9986    fn new(gauges: &[String]) -> Self {
9987        Self {
9988            probes: 0,
9989            gauges: gauges
9990                .iter()
9991                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9992                .collect(),
9993        }
9994    }
9995
9996    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9997    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9998        self.probes = self.probes.saturating_add(1);
9999        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10000            trace.latest = Some((verdict.clone(), at));
10001            if matches!(
10002                verdict,
10003                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10004            ) {
10005                trace.last_value = Some((verdict.clone(), at));
10006            }
10007        }
10008    }
10009}
10010
10011/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10012/// declared gauge, separated by ` | `. A value is shown as its verdict
10013/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10014/// and whether that counted as busy.
10015impl fmt::Display for DrainGaugeTrace {
10016    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10017        if self.gauges.is_empty() {
10018            return f.write_str("none declared");
10019        }
10020        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10021            if index > 0 {
10022                f.write_str(" | ")?;
10023            }
10024            write!(f, "{name}: ")?;
10025            match &trace.latest {
10026                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10027                None => f.write_str("never probed")?,
10028            }
10029            match &trace.last_value {
10030                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10031                None => f.write_str("; no value ever read")?,
10032            }
10033        }
10034        Ok(())
10035    }
10036}
10037
10038pub(crate) fn declared_busy_gauges(
10039    registry: &Registry,
10040    module_id: &str,
10041) -> Result<Vec<String>, SuperviseError> {
10042    busy_gauges_of(
10043        registry
10044            .get_module(module_id)
10045            .map_err(SuperviseError::Registry)?,
10046    )
10047}
10048
10049/// [`declared_busy_gauges`] for the registration a connection holds, in any
10050/// slot: after cutover the incumbent is no longer the id's active
10051/// registration, and its own manifest is the one that names its gauges.
10052fn declared_busy_gauges_for_connection(
10053    registry: &Registry,
10054    connection_id: ConnectionId,
10055) -> Result<Vec<String>, SuperviseError> {
10056    busy_gauges_of(
10057        registry
10058            .get_module_by_connection(connection_id)
10059            .map_err(SuperviseError::Registry)?,
10060    )
10061}
10062
10063fn busy_gauges_of(
10064    registration: Option<crate::registry::ModuleRegistration>,
10065) -> Result<Vec<String>, SuperviseError> {
10066    let Some(registration) = registration else {
10067        return Ok(Vec::new());
10068    };
10069    let Some(self_signals) = registration.manifest.self_signals else {
10070        return Ok(Vec::new());
10071    };
10072
10073    let mut gauges = Vec::new();
10074    for declaration in self_signals {
10075        if declaration.kind != SelfSignalKind::Busy {
10076            continue;
10077        }
10078        match declaration.anchored_to {
10079            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10080                gauges.extend(declared)
10081            }
10082            _ => {
10083                // An invalid Busy anchor is fail-safe: the empty name cannot be
10084                // present in a conforming health report, so this drain stays busy.
10085                gauges.push(String::new());
10086            }
10087        }
10088    }
10089    Ok(gauges)
10090}
10091
10092/// Wait for `endpoint` to have nothing in flight and, when the module declares
10093/// busy gauges, for a health probe to report them quiet. The probe is addressed
10094/// by `scope`: a swap's superseded incumbent must be asked about its own
10095/// gauges, and by module id the probe would reach the promoted candidate.
10096/// While the gauges read busy, the probe repeats every
10097/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10098/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10099/// timeout line.
10100#[allow(clippy::too_many_arguments)]
10101async fn wait_for_forwarding_quiescence(
10102    forwarding: &ForwardingTable,
10103    module_id: &str,
10104    runtime: &SupervisorRuntimeConfig,
10105    endpoint: crate::ModuleEndpointId,
10106    deadline: Instant,
10107    busy_gauges: &[String],
10108    scope: DrainScope,
10109    trace: &mut DrainGaugeTrace,
10110) -> Result<bool, SuperviseError> {
10111    let wait_started = Instant::now();
10112    let mut gauges_quiescent = busy_gauges.is_empty();
10113    let mut next_probe_at = wait_started;
10114    let mut omission_reported = false;
10115
10116    loop {
10117        let now = Instant::now();
10118        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10119            let report = match scope {
10120                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10121                DrainScope::Endpoint(endpoint) => {
10122                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10123                }
10124            };
10125            let verdicts = match &report {
10126                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10127                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10128            };
10129            trace.record(&verdicts, wait_started.elapsed());
10130            gauges_quiescent = match report {
10131                Ok(_) => match busy_gauge_observation(&verdicts) {
10132                    BusyGaugeObservation::Quiescent => true,
10133                    BusyGaugeObservation::Busy => false,
10134                    BusyGaugeObservation::Omitted => {
10135                        // Once per drain: a module that never reports a gauge
10136                        // it declared would otherwise say so on every probe.
10137                        if !omission_reported {
10138                            forwarding
10139                                .counters()
10140                                .increment_drains_with_undeclared_gauge();
10141                            let missing = busy_gauges
10142                                .iter()
10143                                .zip(&verdicts)
10144                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10145                                .map(|(gauge, _)| gauge.as_str())
10146                                .collect::<Vec<_>>()
10147                                .join(",");
10148                            warn!(
10149                                module_id,
10150                                missing_gauges = %missing,
10151                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10152                            );
10153                            omission_reported = true;
10154                        }
10155                        false
10156                    }
10157                },
10158                Err(err) => {
10159                    warn!(
10160                        module_id,
10161                        error = %err,
10162                        "drain health.check did not produce declared busy gauges; treating module as busy"
10163                    );
10164                    false
10165                }
10166            };
10167            // Counted from the probe's completion: the probe above was awaited,
10168            // so no second probe can start while one is still outstanding.
10169            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10170        }
10171
10172        let in_flight = forwarding
10173            .endpoint_in_flight_count(endpoint)
10174            .map_err(SuperviseError::Forwarding)?;
10175        if in_flight == 0 && gauges_quiescent {
10176            return Ok(true);
10177        }
10178
10179        let now = Instant::now();
10180        if now >= deadline {
10181            return Ok(false);
10182        }
10183        let mut wait = deadline
10184            .saturating_duration_since(now)
10185            .min(REGISTRY_RELEASE_POLL);
10186        if !busy_gauges.is_empty() {
10187            wait = wait.min(next_probe_at.saturating_duration_since(now));
10188        }
10189        sleep(wait).await;
10190    }
10191}
10192
10193/// The wait between drain busy-gauge probes: the larger of
10194/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10195/// drain's own in-flight poll interval.
10196fn drain_busy_reprobe_interval() -> Duration {
10197    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10198}
10199
10200/// Logs why a drain gave up: the drain waited its full timeout without the
10201/// module going quiet, and teardown is being forced.
10202///
10203/// The line names two kinds of evidence, because a bare "timed out" hides
10204/// causes that need different fixes:
10205/// - `holdouts`: the requests and routes still open when the timeout hit,
10206///   and on which connections. One stuck call is a module bug; a held stream
10207///   on every session is a module that should end its streams when it
10208///   receives `module.draining`.
10209/// - `gauges`: for each busy gauge the module declares, its latest reading
10210///   and how many probes ran. This separates a module that kept reporting
10211///   itself busy, one whose health answer never contained the gauge, and one
10212///   that never answered the probes, which otherwise all look like a timeout
10213///   with nothing in flight.
10214fn warn_drain_timed_out(
10215    module_id: &str,
10216    drain_timeout: Duration,
10217    reason: RouteCloseReason,
10218    holdouts: &crate::forwarding::DrainHoldouts,
10219    gauges: &DrainGaugeTrace,
10220) {
10221    warn!(
10222        module_id = %module_id,
10223        waited = ?drain_timeout,
10224        ?reason,
10225        held_requests = holdouts.requests,
10226        held_routes = holdouts.routes,
10227        total_routes = holdouts.total_routes,
10228        top_connections = ?holdouts.top_connections,
10229        // `module_channel:corr`, so the module can find each held request
10230        // in its own log; capped, so `held_requests` is the full count.
10231        held = %holdouts
10232            .held
10233            .iter()
10234            .map(|(channel, corr)| format!("{channel}:{corr}"))
10235            .collect::<Vec<_>>()
10236            .join(","),
10237        gauge_probes = gauges.probes,
10238        busy_gauges = %gauges,
10239        "route drain timed out before request quiescence; forcing teardown"
10240    );
10241}
10242
10243/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10244///
10245/// `Ok` is always honest and passed straight through -- the wait actually measured
10246/// in-flight state. `Err` means the wait produced no measurement at all (the
10247/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10248/// constant: the drain did not complete. Never recomputed from route state, never a
10249/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10250fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10251    match wait_result {
10252        Ok(drained) => *drained,
10253        Err(_) => false,
10254    }
10255}
10256
10257fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10258    for released in released_routes {
10259        let frame = match Frame::build_with_version(
10260            released.negotiated_ver,
10261            FrameType::Goodbye,
10262            control_flags(),
10263            released.channel,
10264            released.epoch,
10265            0,
10266            Vec::new(),
10267        ) {
10268            Ok(frame) => frame,
10269            Err(err) => {
10270                warn!(
10271                    route_channel = released.channel,
10272                    error = %err,
10273                    "failed to build supervisor drain route GOODBYE frame"
10274                );
10275                continue;
10276            }
10277        };
10278        if !released.close_on_delivery_failure() {
10279            crate::forwarding::send_module_route_goodbye(
10280                &forwarding.counters(),
10281                &released.sink,
10282                frame,
10283                released.module_id.as_deref(),
10284                "supervisor drain",
10285            );
10286            continue;
10287        }
10288        if let Err(err) = released.sink.try_send(frame) {
10289            warn!(
10290                target_connection_id = released.connection_id.get(),
10291                route_channel = released.channel,
10292                error = %err,
10293                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10294            );
10295            let _ = forwarding.escalate_client_delivery_failure(
10296                released.connection_id,
10297                released.channel,
10298                released.epoch,
10299                CloseReason::new(
10300                    "route_goodbye_delivery_failed",
10301                    format!(
10302                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10303                        released.channel
10304                    ),
10305                ),
10306                crate::forwarding::UndeliveredFrame {
10307                    module_id: released.module_id.as_deref(),
10308                    sink: &released.sink,
10309                },
10310            );
10311        }
10312    }
10313}
10314
10315fn send_module_draining(
10316    module_id: &str,
10317    reason: RouteCloseReason,
10318    deadline_ms: u64,
10319    target: &ModuleDrainTarget,
10320) {
10321    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10322        reason,
10323        deadline_ms,
10324    }) {
10325        Ok(body) => body,
10326        Err(err) => {
10327            warn!(
10328                module_id,
10329                error = %err,
10330                "failed to encode module draining command"
10331            );
10332            return;
10333        }
10334    };
10335    let frame = match Frame::build_with_version(
10336        target.negotiated_ver,
10337        FrameType::Push,
10338        control_flags(),
10339        0,
10340        0,
10341        0,
10342        body,
10343    ) {
10344        Ok(frame) => frame,
10345        Err(err) => {
10346            warn!(
10347                module_id,
10348                error = %err,
10349                "failed to build module draining command frame"
10350            );
10351            return;
10352        }
10353    };
10354    if let Err(err) = target.sink.try_send(frame) {
10355        warn!(
10356            module_id,
10357            target_connection_id = target.endpoint.connection_id.get(),
10358            error = %err,
10359            "module draining command was not delivered to peer"
10360        );
10361    }
10362}
10363
10364/// The channel-0 GOODBYE that tells a module its stop is planned.
10365fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10366    match Frame::build_with_version(
10367        negotiated_ver,
10368        FrameType::Goodbye,
10369        control_flags(),
10370        0,
10371        0,
10372        0,
10373        Vec::new(),
10374    ) {
10375        Ok(frame) => Some(frame),
10376        Err(err) => {
10377            warn!(
10378                module_id,
10379                error = %err,
10380                "failed to build module GOODBYE frame"
10381            );
10382            None
10383        }
10384    }
10385}
10386
10387/// Send every registered module connection its module GOODBYE at daemon
10388/// shutdown, then request that connection's close.
10389///
10390/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10391/// before EOF, so the GOODBYE must reach the socket before the close. A close
10392/// request does not wait for the connection's queued frames: its writer gets a
10393/// bounded grace after the close, is aborted if it overruns it, and the daemon
10394/// process may exit before that grace ends. So with `wait_for_flush`, each
10395/// connection is closed only after its writer has acknowledged writing the
10396/// GOODBYE, or once a short shared budget runs out, so one module that is not
10397/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10398/// are only queued, for a shutdown the operator has told to stop waiting.
10399/// A connection that is already gone is skipped.
10400#[cfg(unix)]
10401async fn send_module_goodbyes_for_daemon_shutdown(
10402    forwarding: &Arc<ForwardingTable>,
10403    reason: &CloseReason,
10404    wait_for_flush: bool,
10405) {
10406    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10407    let targets = match forwarding.module_connections() {
10408        Ok(targets) => targets,
10409        Err(err) => {
10410            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10411            return;
10412        }
10413    };
10414    let deadline = Instant::now() + GOODBYE_BUDGET;
10415    let mut sends = tokio::task::JoinSet::new();
10416    for target in targets {
10417        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10418            continue;
10419        };
10420        if !wait_for_flush {
10421            if let Err(err) = target.sink.try_send(frame) {
10422                debug!(
10423                    module_id = %target.module_id,
10424                    error = %err,
10425                    "shutdown module GOODBYE was not queued"
10426                );
10427            }
10428            continue;
10429        }
10430        let forwarding = Arc::clone(forwarding);
10431        let reason = reason.clone();
10432        sends.spawn(async move {
10433            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10434                Ok(Ok(())) => {}
10435                Ok(Err(err)) => debug!(
10436                    module_id = %target.module_id,
10437                    error = %err,
10438                    "module connection closed before its shutdown GOODBYE was written"
10439                ),
10440                Err(_) => warn!(
10441                    module_id = %target.module_id,
10442                    budget = ?GOODBYE_BUDGET,
10443                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10444                ),
10445            }
10446            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10447        });
10448    }
10449    // Every task ends by the shared deadline, so this wait is bounded too.
10450    while sends.join_next().await.is_some() {}
10451}
10452
10453fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10454    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10455        return;
10456    };
10457    if let Err(err) = target.sink.try_send(frame) {
10458        warn!(
10459            module_id,
10460            target_connection_id = target.endpoint.connection_id.get(),
10461            error = %err,
10462            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10463        );
10464        forwarding.request_connection_close(
10465            target.endpoint.connection_id,
10466            CloseReason::new(
10467                "module_goodbye_delivery_failed",
10468                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10469            ),
10470        );
10471    }
10472}
10473
10474#[derive(Clone, Copy)]
10475struct ForwardingDrainContext<'a> {
10476    spec: &'a ModuleSpec,
10477    runtime: &'a SupervisorRuntimeConfig,
10478    registry: &'a Registry,
10479    scope: DrainScope,
10480}
10481
10482/// Which process a forwarding drain addresses.
10483#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10484enum DrainScope {
10485    /// Whatever endpoint is active for the module id: every plain stop,
10486    /// restart and reload. Also moves the module's state to `Draining`.
10487    Active,
10488    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10489    /// module id would resolve to the promoted candidate and leave neither
10490    /// process routable. The module's state is left alone, since the promoted
10491    /// candidate is what it describes and that process is running.
10492    Endpoint(crate::ModuleEndpointId),
10493}
10494
10495/// Whether a child being drained has already been asked to stop by the time
10496/// its drain wait starts.
10497///
10498/// The drain wait is the same budget whatever this says. What it decides is
10499/// whether the supervisor must ask by signal before that wait begins: a child
10500/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10501/// healthy or not.
10502#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10503enum StopNotice {
10504    /// The module was sent `module.draining` and a module GOODBYE over its own
10505    /// registered connection, and stops itself.
10506    SentOverConnection,
10507    /// The forwarding drain found no registered connection for the module: a
10508    /// subc child spawned moments ago that has not sent HELLO yet, or a
10509    /// `protocol: "none"` child, which never registers.
10510    NoConnection,
10511    /// This path sends nothing over the module's connection: the supervisor has
10512    /// no forwarding table, or the caller stops the child without a forwarding
10513    /// drain.
10514    NotSent,
10515}
10516
10517async fn begin_forwarding_drain(
10518    spec: &ModuleSpec,
10519    runtime: &SupervisorRuntimeConfig,
10520    registry: &Registry,
10521    snapshot: &SharedSnapshot,
10522    enabled: Option<bool>,
10523    reason: RouteCloseReason,
10524) -> Result<StopNotice, SuperviseError> {
10525    let Some(forwarding) = runtime.forwarding.as_ref() else {
10526        return Err(SuperviseError::ReloadUnavailable {
10527            module_id: spec.module_id.clone(),
10528            reason: "supervisor was not configured with a forwarding table".to_string(),
10529        });
10530    };
10531
10532    begin_forwarding_drain_with(
10533        forwarding,
10534        ForwardingDrainContext {
10535            spec,
10536            runtime,
10537            registry,
10538            scope: DrainScope::Active,
10539        },
10540        snapshot,
10541        enabled,
10542        reason,
10543        runtime.drain_timeout,
10544    )
10545    .await
10546}
10547
10548async fn begin_forwarding_drain_if_configured(
10549    spec: &ModuleSpec,
10550    runtime: &SupervisorRuntimeConfig,
10551    registry: &Registry,
10552    snapshot: &SharedSnapshot,
10553    enabled: Option<bool>,
10554    reason: RouteCloseReason,
10555) -> Result<StopNotice, SuperviseError> {
10556    begin_forwarding_drain_with_timeout(
10557        spec,
10558        runtime,
10559        registry,
10560        snapshot,
10561        enabled,
10562        reason,
10563        runtime.drain_timeout,
10564    )
10565    .await
10566}
10567
10568/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10569/// budget, for paths where the operator overrides the module's configured one
10570/// (`supervisor.restart{drain_timeout_ms}`).
10571async fn begin_forwarding_drain_with_timeout(
10572    spec: &ModuleSpec,
10573    runtime: &SupervisorRuntimeConfig,
10574    registry: &Registry,
10575    snapshot: &SharedSnapshot,
10576    enabled: Option<bool>,
10577    reason: RouteCloseReason,
10578    drain_timeout: Duration,
10579) -> Result<StopNotice, SuperviseError> {
10580    let Some(forwarding) = runtime.forwarding.as_ref() else {
10581        return Ok(StopNotice::NotSent);
10582    };
10583
10584    begin_forwarding_drain_with(
10585        forwarding,
10586        ForwardingDrainContext {
10587            spec,
10588            runtime,
10589            registry,
10590            scope: DrainScope::Active,
10591        },
10592        snapshot,
10593        enabled,
10594        reason,
10595        drain_timeout,
10596    )
10597    .await
10598}
10599
10600async fn begin_forwarding_drain_with(
10601    forwarding: &ForwardingTable,
10602    context: ForwardingDrainContext<'_>,
10603    snapshot: &SharedSnapshot,
10604    enabled: Option<bool>,
10605    reason: RouteCloseReason,
10606    drain_timeout: Duration,
10607) -> Result<StopNotice, SuperviseError> {
10608    let ForwardingDrainContext {
10609        spec,
10610        runtime,
10611        registry,
10612        scope,
10613    } = context;
10614    debug_assert_ne!(reason, RouteCloseReason::Crash);
10615    let terminal = matches!(reason, RouteCloseReason::Disable);
10616    let drain_started_at = Instant::now();
10617    let drain_deadline = drain_started_at + drain_timeout;
10618    let deadline_ms =
10619        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10620    let busy_gauges = match scope {
10621        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10622        DrainScope::Endpoint(endpoint) => {
10623            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10624        }
10625    };
10626
10627    // Admission gate first: route.open/commit and route REQUEST admission are closed
10628    // before the first quiescence check, so the outstanding count can only fall.
10629    let gate_started = Instant::now();
10630    let drain_target = match scope {
10631        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10632        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10633    }
10634    .map_err(SuperviseError::Forwarding)?;
10635    // The instant admission closed, and how long taking the forwarding write
10636    // lock to close it took. The timeout line reports only the quiescence
10637    // wait, so without this a drain that started late looked like one that
10638    // started on time.
10639    info!(
10640        module_id = %spec.module_id,
10641        ?reason,
10642        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10643        connected = drain_target.is_some(),
10644        "module drain began; route admission closed"
10645    );
10646    if scope == DrainScope::Active {
10647        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10648            state.state = ModuleState::Draining;
10649            state.draining_to_replace =
10650                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10651            if let Some(enabled) = enabled {
10652                state.enabled = enabled;
10653            }
10654        })?;
10655    }
10656
10657    let Some(target) = drain_target.as_ref() else {
10658        // Nothing was sent: the module has no registered connection to carry
10659        // `module.draining` or a GOODBYE. The caller must not assume the child
10660        // was asked to stop.
10661        return Ok(StopNotice::NoConnection);
10662    };
10663    {
10664        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10665        #[cfg(feature = "test-support")]
10666        forwarding.wait_before_route_closing_for_test().await;
10667        let routes = forwarding
10668            .endpoint_routes(target.endpoint)
10669            .map_err(SuperviseError::Forwarding)?;
10670        let routes_notified = routes.len();
10671        crate::control::send_route_control_pushes(
10672            forwarding,
10673            routes.clone(),
10674            ClientControlPush::RouteClosing {
10675                module_id: spec.module_id.clone(),
10676                channels: Vec::new(),
10677                reason,
10678            },
10679        );
10680        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10681
10682        // `route.closing` was just sent above: from here on every return path,
10683        // including an early one, MUST send `route.closed` before propagating
10684        // anything else. A client holds `closing` as a promise that a verdict is
10685        // coming; leaving early without `closed` strands it waiting forever, since
10686        // `closing` carries no timeout of its own.
10687        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10688        let wait_result = wait_for_forwarding_quiescence(
10689            forwarding,
10690            &spec.module_id,
10691            runtime,
10692            target.endpoint,
10693            drain_deadline,
10694            &busy_gauges,
10695            scope,
10696            &mut gauge_trace,
10697        )
10698        .await;
10699        let drained = drained_after_quiescence_wait(&wait_result);
10700        if let Err(err) = &wait_result {
10701            error!(
10702                module_id = %spec.module_id,
10703                ?reason,
10704                error = %err,
10705                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10706            );
10707        } else if !drained {
10708            // Read before teardown releases the routes.
10709            let holdouts = forwarding
10710                .endpoint_drain_holdouts(target.endpoint)
10711                .unwrap_or_default();
10712            warn_drain_timed_out(
10713                &spec.module_id,
10714                drain_timeout,
10715                reason,
10716                &holdouts,
10717                &gauge_trace,
10718            );
10719        }
10720        crate::control::send_route_control_pushes(
10721            forwarding,
10722            routes,
10723            ClientControlPush::RouteClosed {
10724                module_id: spec.module_id.clone(),
10725                channels: Vec::new(),
10726                reason,
10727                drained,
10728                abandoned: target.abandoned_bindings.len() as u32,
10729                excluded_subscriptions: target.excluded_subscriptions,
10730                terminal: Some(terminal),
10731            },
10732        );
10733        wait_result?;
10734
10735        // `route.closed` has now been sent unconditionally above. From here the
10736        // remaining steps are cleanup (route + module GOODBYE) rather than a
10737        // promise the client is waiting on, but a lock-poisoned
10738        // `release_module_endpoint_routes` would otherwise skip the module
10739        // GOODBYE silently too -- send it before propagating the error.
10740        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10741            Ok(routes) => routes,
10742            Err(err) => {
10743                warn!(
10744                    module_id = %spec.module_id,
10745                    ?reason,
10746                    error = %err,
10747                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10748                );
10749                send_module_goodbye(&spec.module_id, forwarding, target);
10750                return Err(SuperviseError::Forwarding(err));
10751            }
10752        };
10753        let route_goodbye_count = released_routes.len();
10754        send_route_goodbyes(forwarding, released_routes);
10755        send_module_goodbye(&spec.module_id, forwarding, target);
10756
10757        // The drain's happy path was previously silent: every emission above is
10758        // best-effort with only its failure arm logged, so "were consumers told"
10759        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10760        // hang where the open question was exactly whether teardown notice went
10761        // out). One summary line makes that class decidable in one grep.
10762        info!(
10763            module_id = %spec.module_id,
10764            ?reason,
10765            routes_notified,
10766            route_goodbyes = route_goodbye_count,
10767            abandoned_reservations = target.abandoned_bindings.len(),
10768            excluded_subscriptions = target.excluded_subscriptions,
10769            drained,
10770            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10771        );
10772    }
10773
10774    Ok(StopNotice::SentOverConnection)
10775}
10776
10777/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10778/// the only slot a plain (non-swap) spawn can register into.
10779async fn wait_for_registration_after_reload(
10780    registry: &Registry,
10781    module_id: &str,
10782    snapshot: &SharedSnapshot,
10783    child: &mut SupervisedChild,
10784    wait: Duration,
10785) -> Result<RegistrationWaitOutcome, SuperviseError> {
10786    wait_for_slot_registration(
10787        registry,
10788        crate::registry::RegistrationSlot::Active(module_id),
10789        module_id,
10790        snapshot,
10791        child,
10792        wait,
10793    )
10794    .await
10795}
10796
10797/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10798///
10799/// Keyed on the slot rather than the bare module id because during a swap the
10800/// id's active slot is already held by the incumbent: an id-keyed wait would
10801/// report the incumbent's registration as the candidate's and a candidate that
10802/// never registers would look registered. A swap candidate waits on
10803/// `crate::registry::RegistrationSlot::Candidate`.
10804async fn wait_for_slot_registration(
10805    registry: &Registry,
10806    slot: crate::registry::RegistrationSlot<'_>,
10807    module_id: &str,
10808    snapshot: &SharedSnapshot,
10809    child: &mut SupervisedChild,
10810    wait: Duration,
10811) -> Result<RegistrationWaitOutcome, SuperviseError> {
10812    let deadline = Instant::now() + wait;
10813    loop {
10814        if registry
10815            .registration(slot)
10816            .map_err(SuperviseError::Registry)?
10817            .is_some()
10818        {
10819            return Ok(RegistrationWaitOutcome::Registered);
10820        }
10821
10822        let now = Instant::now();
10823        if now >= deadline {
10824            return Ok(RegistrationWaitOutcome::TimedOut);
10825        }
10826        let remaining = deadline.saturating_duration_since(now);
10827        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10828
10829        tokio::select! {
10830            wait_result = child.wait() => {
10831                let status = wait_result.map_err(|source| SuperviseError::Wait {
10832                    module_id: module_id.to_string(),
10833                    source,
10834                })?;
10835                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10836                    snapshot,
10837                    child,
10838                    &status,
10839                )));
10840            }
10841            _ = sleep(poll) => {}
10842        }
10843    }
10844}
10845
10846fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10847    // A replacement process that exits before HELLO did not provide service, even
10848    // if it used status 0. Count it against the restart cap as a new-binary failure.
10849    if exit_report.kind != ExitKind::DeliberateSeverance {
10850        exit_report.kind = ExitKind::Crash;
10851    }
10852    exit_report
10853}
10854
10855async fn handle_reload_child_registration_failure(
10856    spec: &ModuleSpec,
10857    runtime: &SupervisorRuntimeConfig,
10858    registry: &Registry,
10859    process_liveness: &SupervisorProcessLiveness,
10860    snapshot: &SharedSnapshot,
10861    _child: &mut Option<SupervisedChild>,
10862    failure: ReloadRegistrationFailure,
10863) -> Result<(), SuperviseError> {
10864    let ReloadRegistrationFailure {
10865        exit_report,
10866        reason,
10867    } = failure;
10868    match on_child_exit(
10869        spec,
10870        runtime.restart_policy,
10871        registry,
10872        snapshot,
10873        &runtime.terminal_ring,
10874        &runtime.spawn_events,
10875        &runtime.child_roster,
10876        exit_report,
10877        runtime.recovery_is_daemon,
10878    )
10879    .await
10880    {
10881        NextAction::Stop {
10882            registration_released,
10883        } => {
10884            if registration_released {
10885                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10886            }
10887        }
10888        NextAction::Restart { schedule } => {
10889            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10890                schedule.delay
10891            });
10892            if let Some(schedule) = schedule {
10893                log_crash_respawn(&spec.module_id, schedule);
10894            }
10895            schedule_respawn(
10896                runtime,
10897                snapshot,
10898                &spec.module_id,
10899                delay,
10900                RespawnKind::Spawn,
10901                true,
10902            )?;
10903        }
10904    }
10905    Err(SuperviseError::ReloadFailed {
10906        module_id: spec.module_id.clone(),
10907        reason,
10908    })
10909}
10910
10911async fn handle_reload_spawn_failure(
10912    spec: &ModuleSpec,
10913    runtime: &SupervisorRuntimeConfig,
10914    process_liveness: &SupervisorProcessLiveness,
10915    snapshot: &SharedSnapshot,
10916    _child: &mut Option<SupervisedChild>,
10917    reason: String,
10918) -> Result<(), SuperviseError> {
10919    let now = Instant::now();
10920    let mut schedule = None;
10921    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10922        clear_current_process_facts(state);
10923        if state.enabled {
10924            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10925            state.state = if schedule.is_some() {
10926                ModuleState::Restarting
10927            } else {
10928                if runtime
10929                    .failure_context(runtime.recovery_is_daemon)
10930                    .may_retry()
10931                {
10932                    state.failed_retry.schedule(&spec.module_id);
10933                }
10934                ModuleState::Failed
10935            };
10936        } else {
10937            state.state = ModuleState::Disabled;
10938        }
10939    })?;
10940    if let Some(schedule) = schedule {
10941        schedule_respawn(
10942            runtime,
10943            snapshot,
10944            &spec.module_id,
10945            schedule.delay,
10946            RespawnKind::Spawn,
10947            true,
10948        )?;
10949    } else {
10950        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10951    }
10952    Err(SuperviseError::ReloadFailed {
10953        module_id: spec.module_id.clone(),
10954        reason,
10955    })
10956}
10957
10958fn control_flags() -> Flags {
10959    Flags::new(false, Priority::Passive, false)
10960}
10961
10962#[allow(clippy::too_many_arguments)]
10963async fn drain_optional_child(
10964    module_id: &str,
10965    protocol: ModuleProtocol,
10966    stop_notice: StopNotice,
10967    registry: &Registry,
10968    forwarding: Option<&ForwardingTable>,
10969    snapshot: &SharedSnapshot,
10970    terminal_ring: &Arc<Mutex<TerminalRing>>,
10971    spawn_events: &SpawnEventFeed,
10972    child: &mut Option<SupervisedChild>,
10973    drain_timeout: Duration,
10974    final_state: ModuleState,
10975    enabled: Option<bool>,
10976    failure: &FailureContext,
10977) -> Result<(), SuperviseError> {
10978    if let Some(child) = child.take() {
10979        drain_child_to_state(
10980            module_id,
10981            protocol,
10982            stop_notice,
10983            registry,
10984            forwarding,
10985            snapshot,
10986            terminal_ring,
10987            spawn_events,
10988            child,
10989            drain_timeout,
10990            final_state,
10991            enabled,
10992            failure,
10993        )
10994        .await
10995    } else {
10996        update_snapshot(snapshot, Some(module_id), |state| {
10997            state.state = final_state;
10998            if final_state == ModuleState::Failed {
10999                state.failed_retry.failed_at = Some(Instant::now());
11000            }
11001            if let Some(enabled) = enabled {
11002                state.enabled = enabled;
11003            }
11004            clear_current_process_facts(state);
11005        })?;
11006        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11007    }
11008}
11009
11010#[allow(clippy::too_many_arguments)]
11011async fn drain_child_to_state(
11012    module_id: &str,
11013    _protocol: ModuleProtocol,
11014    stop_notice: StopNotice,
11015    registry: &Registry,
11016    forwarding: Option<&ForwardingTable>,
11017    snapshot: &SharedSnapshot,
11018    terminal_ring: &Arc<Mutex<TerminalRing>>,
11019    spawn_events: &SpawnEventFeed,
11020    mut child: SupervisedChild,
11021    drain_timeout: Duration,
11022    final_state: ModuleState,
11023    enabled: Option<bool>,
11024    failure: &FailureContext,
11025) -> Result<(), SuperviseError> {
11026    let protocol = child.protocol;
11027    update_snapshot(snapshot, Some(module_id), |state| {
11028        state.state = ModuleState::Draining;
11029        state.draining_to_replace = final_state == ModuleState::Restarting;
11030        if let Some(enabled) = enabled {
11031            state.enabled = enabled;
11032        }
11033    })?;
11034
11035    // The wait below is the same budget in every case; what differs is
11036    // whether anything has ASKED the child to stop before it starts. Only a
11037    // forwarding drain that reached the module's registered connection has
11038    // (`module.draining`, then a module GOODBYE). Every other child was told
11039    // nothing: a `protocol: "none"` module, which never registers; a subc
11040    // module spawned moments ago that has not sent HELLO yet; or a stop that
11041    // runs no forwarding drain. Without a signal the budget is only a delay
11042    // in front of SIGKILL -- and the not-yet-registered child is the worst
11043    // case, because it registers into a module that is already draining,
11044    // is never told, and is killed while healthy.
11045    if stop_notice != StopNotice::SentOverConnection {
11046        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11047            info!(
11048                module_id,
11049                pid = child.pid,
11050                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11051                "module has no connection yet; requesting stop by signal"
11052            );
11053        }
11054        request_graceful_stop(module_id, &child);
11055    }
11056
11057    let exit_report = match timeout(drain_timeout, child.wait()).await {
11058        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11059        Ok(Err(source)) => {
11060            fail_after_wait_error(
11061                module_id,
11062                snapshot,
11063                terminal_ring,
11064                spawn_events,
11065                format!("failed to wait during drain: {source}"),
11066                failure,
11067            );
11068            return Err(SuperviseError::Wait {
11069                module_id: module_id.to_string(),
11070                source,
11071            });
11072        }
11073        Err(_) => {
11074            // Mirror the sibling arm above: state is already `Draining`, and an
11075            // error propagated from here would strand it there -- a state
11076            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11077            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11078            // `Failed` before `?` keeps the module operator-visible and
11079            // revivable. Trigger is an ESRCH race (process exits between the
11080            // drain timeout firing and the kill) or a post-kill wait failure
11081            // (issue #34).
11082            //
11083            // Logged because the kill is otherwise visible only as signal 9 in
11084            // the terminal ring, and the budget it follows can be long enough
11085            // that consumers see a stretch of refusals with no stated cause.
11086            warn!(
11087                module_id,
11088                pid = child.pid,
11089                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11090                reason = ?final_state,
11091                ?stop_notice,
11092                "drain budget expired before the module exited; killing it"
11093            );
11094            child.start_kill().map_err(|source| {
11095                fail_after_wait_error(
11096                    module_id,
11097                    snapshot,
11098                    terminal_ring,
11099                    spawn_events,
11100                    format!("failed to kill during drain: {source}"),
11101                    failure,
11102                );
11103                SuperviseError::Kill {
11104                    module_id: module_id.to_string(),
11105                    source,
11106                }
11107            })?;
11108            let status = child.wait().await.map_err(|source| {
11109                fail_after_wait_error(
11110                    module_id,
11111                    snapshot,
11112                    terminal_ring,
11113                    spawn_events,
11114                    format!("failed to wait after drain kill: {source}"),
11115                    failure,
11116                );
11117                SuperviseError::Wait {
11118                    module_id: module_id.to_string(),
11119                    source,
11120                }
11121            })?;
11122            classify_reaped_child_exit(snapshot, &child, &status)
11123        }
11124    };
11125
11126    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11127    publish_after_terminal_record(
11128        module_id,
11129        || {
11130            record_terminal_with_detail(
11131                module_id,
11132                terminal_ring,
11133                spawn_events,
11134                &exit_report,
11135                terminal_disposition(final_state),
11136                detail,
11137            )
11138        },
11139        || {
11140            update_snapshot(snapshot, Some(module_id), |state| {
11141                state.state = final_state;
11142                if final_state == ModuleState::Failed {
11143                    state.failed_retry.failed_at = Some(Instant::now());
11144                }
11145                if let Some(enabled) = enabled {
11146                    state.enabled = enabled;
11147                }
11148                clear_current_process_facts(state);
11149                state.last_exit = Some(exit_report.clone());
11150                if exit_report.kind == ExitKind::DeliberateSeverance {
11151                    state.lifetime_restarts += 1;
11152                }
11153            })
11154        },
11155    )?;
11156    child.drain_stderr(module_id).await;
11157
11158    release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11159}
11160
11161/// Ask a child that nothing else has asked to stop, by signal.
11162///
11163/// A registered subc module is asked over its own connection: the drain sends
11164/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11165/// module GOODBYE, and the module stops itself. A module that speaks no subc
11166/// wire receives none of that, and neither does a subc module that has not
11167/// registered yet, so for them the drain budget would be pure delay in front of
11168/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11169/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11170/// into a recovery on the next start.
11171///
11172/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11173/// rule rather than an optimisation: that module's graceful stop is already
11174/// running by the time its child is drained, and a signal would race it.
11175///
11176/// Best-effort by construction. A child that has already exited is the ordinary
11177/// case rather than an error (the kill lands on a reaped or exiting pid), so a
11178/// failure is logged at debug and the wait-then-kill below still decides the
11179/// outcome.
11180#[cfg(unix)]
11181fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11182    let Some(pid) = child
11183        .id()
11184        .and_then(|pid| i32::try_from(pid).ok())
11185        .and_then(rustix::process::Pid::from_raw)
11186    else {
11187        debug!(
11188            module_id,
11189            "no pid to signal for teardown; falling through to the drain wait"
11190        );
11191        return;
11192    };
11193    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11194        Ok(()) => debug!(
11195            module_id,
11196            "sent SIGTERM to a module nothing else asked to stop"
11197        ),
11198        Err(err) => debug!(
11199            module_id,
11200            error = %err,
11201            "SIGTERM to module failed; the drain wait and kill still apply"
11202        ),
11203    }
11204}
11205
11206/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11207/// Windows does offer need cooperation this supervisor cannot assume: a console
11208/// control event requires sharing a console with the child, and `WM_CLOSE`
11209/// requires the child to pump a message loop. A supervised server process does
11210/// neither, so there is nothing to send and teardown is the wait followed by the
11211/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11212/// the thing `protocol: "none"` exists to avoid.
11213#[cfg(not(unix))]
11214fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11215    debug!(
11216        module_id,
11217        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11218    );
11219}
11220
11221fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11222    match final_state {
11223        ModuleState::Stopped => TerminalDisposition::Stopped,
11224        ModuleState::Disabled => TerminalDisposition::Disabled,
11225        ModuleState::Restarting => TerminalDisposition::Restarting,
11226        ModuleState::Failed => TerminalDisposition::Failed,
11227        ModuleState::Starting
11228        | ModuleState::Running
11229        | ModuleState::Unresponsive
11230        | ModuleState::Draining => {
11231            unreachable!("terminal exits only finish in terminal or restarting states")
11232        }
11233    }
11234}
11235
11236/// Release a reaped child's registration before allowing another spawn.
11237///
11238/// EOF is not a process-lifetime signal: an inherited socket can stay open
11239/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11240/// reading EOF. After the normal release grace, request connection close (which
11241/// cancels both reads and dispatch), then allow one more release grace for the
11242/// connection guard's forwarding cleanup. Never evict a different connection.
11243async fn release_dead_registration(
11244    registry: &Registry,
11245    forwarding: Option<&ForwardingTable>,
11246    snapshot: &SharedSnapshot,
11247    module_id: &str,
11248    failure: &FailureContext,
11249) -> Result<(), SuperviseError> {
11250    let result = async {
11251        let registration = registry
11252            .get_module(module_id)
11253            .map_err(SuperviseError::Registry)?;
11254        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11255            Ok(()) => return Ok(()),
11256            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11257            Err(err) => return Err(err),
11258        }
11259        let pid = lock_snapshot(snapshot)?.reaped_pid;
11260        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11261            warn!(
11262                module_id,
11263                pid,
11264                connection_id = registration.connection_id.get(),
11265                "reaped module registration outlived release grace; closing dead connection"
11266            );
11267            forwarding.request_connection_close(
11268                registration.connection_id,
11269                CloseReason::new(
11270                    "supervised_process_reaped",
11271                    format!("module '{module_id}' pid {pid} exited"),
11272                ),
11273            );
11274            wait_for_slot_registration_release(
11275                registry,
11276                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11277                REGISTRY_RELEASE_TIMEOUT,
11278            )
11279            .await?;
11280        }
11281        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11282    }
11283    .await;
11284    if let Err(err) = &result {
11285        fail_snapshot(snapshot, Some(module_id), None, failure);
11286        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11287    }
11288    result
11289}
11290
11291/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11292/// plain stop or restart waits for before it spawns a replacement.
11293async fn wait_for_registration_release(
11294    registry: &Registry,
11295    module_id: &str,
11296    wait: Duration,
11297) -> Result<(), SuperviseError> {
11298    wait_for_slot_registration_release(
11299        registry,
11300        crate::registry::RegistrationSlot::Active(module_id),
11301        wait,
11302    )
11303    .await
11304}
11305
11306/// Wait for the registration in `slot` to go away.
11307///
11308/// Keyed on the slot rather than the bare module id because a successful swap
11309/// never empties the id's active slot (the promoted candidate is in it), so an
11310/// id-keyed wait for the incumbent's release would always time out. Draining a
11311/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11312/// incumbent's connection instead.
11313async fn wait_for_slot_registration_release(
11314    registry: &Registry,
11315    slot: crate::registry::RegistrationSlot<'_>,
11316    wait: Duration,
11317) -> Result<(), SuperviseError> {
11318    let deadline = Instant::now() + wait;
11319    let mut release_events = registration_release_events().subscribe();
11320    let still_active = |registration: &crate::registry::ModuleRegistration| {
11321        SuperviseError::RegistrationStillActive {
11322            module_id: registration.manifest.module_id.clone(),
11323            waited: wait,
11324        }
11325    };
11326    loop {
11327        let _observed_generation = *release_events.borrow_and_update();
11328        let Some(registration) = registry
11329            .registration(slot)
11330            .map_err(SuperviseError::Registry)?
11331        else {
11332            return Ok(());
11333        };
11334
11335        let now = Instant::now();
11336        if now >= deadline {
11337            return Err(still_active(&registration));
11338        }
11339
11340        let remaining = deadline.saturating_duration_since(now);
11341        match timeout(remaining, release_events.changed()).await {
11342            Ok(Ok(())) | Ok(Err(_)) => {}
11343            Err(_) => return Err(still_active(&registration)),
11344        }
11345    }
11346}
11347
11348#[cfg(test)]
11349mod slot_registration_wait_tests {
11350    use super::*;
11351    use crate::registry::{ConnectionId, RegistrationSlot};
11352    use subc_protocol::manifest::ModuleManifest;
11353
11354    #[tokio::test]
11355    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11356        let registry = Arc::new(Registry::default());
11357        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11358        let runtime = supervisor.runtime_config();
11359        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11360        let spec = ModuleSpec {
11361            module_id: "enable-stale-registration".to_string(),
11362            program: PathBuf::from("/missing/enable-retry-test"),
11363            args: Vec::new(),
11364            env: Vec::new(),
11365            reserved: false,
11366            reserved_prefixes: Vec::new(),
11367            protocol: ModuleProtocol::Subc,
11368            overlap: Default::default(),
11369        };
11370        let connection = ConnectionId::new(90);
11371        registry
11372            .register_with_control_ops(
11373                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11374                1,
11375                connection,
11376                Vec::new(),
11377            )
11378            .unwrap();
11379        let mut child = None;
11380        let err = set_child_enabled(
11381            &spec,
11382            &runtime,
11383            &registry,
11384            &supervisor.process_liveness,
11385            &snapshot,
11386            &mut child,
11387            true,
11388            &runtime.failure_context(false),
11389        )
11390        .await
11391        .unwrap_err();
11392        assert!(matches!(
11393            err,
11394            SuperviseError::RegistrationStillActive { .. }
11395        ));
11396        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11397        assert!(child.is_none());
11398        registry.deregister_connection(connection).unwrap();
11399        let err = set_child_enabled(
11400            &spec,
11401            &runtime,
11402            &registry,
11403            &supervisor.process_liveness,
11404            &snapshot,
11405            &mut child,
11406            true,
11407            &runtime.failure_context(false),
11408        )
11409        .await
11410        .unwrap_err();
11411        assert!(
11412            matches!(err, SuperviseError::Spawn { .. }),
11413            "second enable must attempt a spawn: {err}"
11414        );
11415        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11416    }
11417
11418    const INCUMBENT: u64 = 1;
11419    const CANDIDATE: u64 = 2;
11420
11421    fn swapped_registry() -> Arc<Registry> {
11422        let registry = Arc::new(Registry::default());
11423        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11424        registry
11425            .register_with_control_ops(
11426                manifest.clone(),
11427                1,
11428                ConnectionId::new(INCUMBENT),
11429                Vec::new(),
11430            )
11431            .unwrap();
11432        registry
11433            .register_candidate_with_control_ops(
11434                manifest,
11435                1,
11436                ConnectionId::new(CANDIDATE),
11437                Vec::new(),
11438            )
11439            .unwrap();
11440        registry
11441    }
11442
11443    /// After a promotion the id's active slot is held by the new process, so an
11444    /// id-keyed wait for the incumbent's release can never succeed; the
11445    /// connection-keyed wait completes as soon as the incumbent deregisters.
11446    #[tokio::test]
11447    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11448        let registry = swapped_registry();
11449        registry.promote_candidate("m").unwrap().unwrap();
11450
11451        assert!(matches!(
11452            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11453            Err(SuperviseError::RegistrationStillActive { .. })
11454        ));
11455
11456        // Still held while the incumbent's connection has not deregistered.
11457        assert!(matches!(
11458            wait_for_slot_registration_release(
11459                &registry,
11460                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11461                Duration::from_millis(50),
11462            )
11463            .await,
11464            Err(SuperviseError::RegistrationStillActive { .. })
11465        ));
11466
11467        let releaser = Arc::clone(&registry);
11468        let release = tokio::spawn(async move {
11469            sleep(Duration::from_millis(20)).await;
11470            releaser
11471                .deregister_connection(ConnectionId::new(INCUMBENT))
11472                .unwrap();
11473            notify_registration_release();
11474        });
11475        wait_for_slot_registration_release(
11476            &registry,
11477            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11478            Duration::from_secs(5),
11479        )
11480        .await
11481        .expect("the incumbent's own registration is released");
11482        release.await.unwrap();
11483        assert!(registry.get_module("m").unwrap().is_some());
11484    }
11485
11486    /// The candidate slot is waited on separately from the active slot: the
11487    /// incumbent's registration neither holds up nor stands in for it.
11488    #[tokio::test]
11489    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11490        let registry = swapped_registry();
11491        assert!(matches!(
11492            wait_for_slot_registration_release(
11493                &registry,
11494                RegistrationSlot::Candidate("m"),
11495                Duration::from_millis(50),
11496            )
11497            .await,
11498            Err(SuperviseError::RegistrationStillActive { .. })
11499        ));
11500        registry
11501            .deregister_connection(ConnectionId::new(CANDIDATE))
11502            .unwrap();
11503        wait_for_slot_registration_release(
11504            &registry,
11505            RegistrationSlot::Candidate("m"),
11506            Duration::from_millis(50),
11507        )
11508        .await
11509        .expect("a candidate slot with no candidate is released");
11510        assert!(registry
11511            .registration(RegistrationSlot::Active("m"))
11512            .unwrap()
11513            .is_some());
11514    }
11515}
11516
11517fn classify_exit(status: &ExitStatus) -> ExitReport {
11518    ExitReport {
11519        kind: if status.success() {
11520            ExitKind::Clean
11521        } else {
11522            ExitKind::Crash
11523        },
11524        code: status.code(),
11525        signal: exit_signal(status),
11526        at_ms: unix_ms_now(),
11527    }
11528}
11529
11530/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11531/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11532/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11533/// disposition still must be `Failed` so the terminal ring is not silently missing
11534/// an entry, matching what `fail_snapshot` records for this same arm.
11535fn wait_error_exit_report() -> ExitReport {
11536    ExitReport {
11537        kind: ExitKind::Crash,
11538        code: None,
11539        signal: None,
11540        at_ms: unix_ms_now(),
11541    }
11542}
11543
11544#[cfg(unix)]
11545fn exit_signal(status: &ExitStatus) -> Option<i32> {
11546    use std::os::unix::process::ExitStatusExt;
11547
11548    status.signal()
11549}
11550
11551#[cfg(not(unix))]
11552fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11553    None
11554}
11555
11556/// Give an operator-touched module its full crash budget back.
11557///
11558/// Named for the counter it used to zero; it now empties the in-window ring,
11559/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11560/// ledger of what happened survives every operator action.
11561fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11562    update_snapshot(snapshot, Some(module_id), |state| {
11563        state.clear_crash_restarts();
11564    })
11565}
11566
11567fn set_running(
11568    snapshot: &SharedSnapshot,
11569    child: &SupervisedChild,
11570    module_id: &str,
11571    spawn_events: &SpawnEventFeed,
11572) -> Result<(), SuperviseError> {
11573    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11574        module_id: Some(module_id.to_string()),
11575    })?;
11576    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11577    if std::mem::take(&mut state.coalesced_restart_pending) {
11578        let generation = state.spawn_generation;
11579        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11580    }
11581    state.drain_disposition_detail = None;
11582    state.spawn_failure = None;
11583    state.waiting_for_previous_pid = None;
11584    state.waiting_for_previous_detail = None;
11585    // Every caller of this is a plain spawn, which always uses the primary key;
11586    // a promoted swap candidate sets the flag itself after this returns.
11587    state.in_alternate_slot = false;
11588    state.configuration_updated_since_spawn = false;
11589    state.spawned_protocol = Some(child.protocol);
11590    state.state = ModuleState::Running;
11591    state.enabled = true;
11592    state.process_alive = true;
11593    state.pid = child.id();
11594    // Probe samples belong to one process. Keep module-level escalation and
11595    // late-answer history, but never charge a new child for its predecessor's misses.
11596    state.health.status = SupervisorHealthStatus::Unknown;
11597    state.health.consecutive_failures = 0;
11598    state.health.detail = None;
11599    state.health.metrics = None;
11600    state.health.last_probe_ms = None;
11601    state.failed_retry.healthy_since = None;
11602    state.failed_retry.failed_at = None;
11603    state.failed_retry.running_since = Some(child.spawned_at);
11604    #[cfg(windows)]
11605    {
11606        state.pipe_consumed = child
11607            .nonce_delivery
11608            .as_ref()
11609            .map(|delivery| delivery.consumed());
11610    }
11611    #[cfg(target_os = "macos")]
11612    {
11613        state.report_ready = Some(Arc::clone(&child.report_ready));
11614    }
11615    state.spawned_at_ms = Some(child.spawned_at_ms);
11616    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11617    state.spawned_from = Some(child.spawned_from.clone());
11618    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11619    state.spawned_file_identity = child.spawned_file_identity.clone();
11620    state.process_start_time = child.process_start_time;
11621    Ok(())
11622}
11623
11624fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11625    state.failed_retry.healthy_since = None;
11626    state.failed_retry.running_since = None;
11627    state.waiting_for_previous_pid = None;
11628    state.waiting_for_previous_detail = None;
11629    state.process_alive = false;
11630    state.spawned_protocol = None;
11631    state.pid = None;
11632    #[cfg(windows)]
11633    {
11634        state.pipe_consumed = None;
11635    }
11636    #[cfg(target_os = "macos")]
11637    {
11638        state.report_ready = None;
11639    }
11640    state.spawned_at_ms = None;
11641    state.exec_outcome = None;
11642    state.spawned_from = None;
11643    state.launch_spec_digest = None;
11644    state.spawned_file_identity = None;
11645    state.process_start_time = None;
11646    state.deliberate_severance = None;
11647}
11648
11649#[cfg(test)]
11650fn record_deliberate_severance(
11651    snapshot: &SharedSnapshot,
11652    identity: ProcessIdentity,
11653) -> Result<(), SuperviseError> {
11654    update_snapshot(snapshot, None, |state| {
11655        state.deliberate_severance = Some(identity);
11656    })
11657}
11658
11659fn apply_deliberate_severance_marker(
11660    snapshot: &SharedSnapshot,
11661    exited_identity: Option<ProcessIdentity>,
11662    mut exit_report: ExitReport,
11663) -> ExitReport {
11664    let marker = lock_snapshot(snapshot)
11665        .ok()
11666        .and_then(|mut state| state.deliberate_severance.take());
11667    if marker.is_some() && marker == exited_identity {
11668        exit_report.kind = ExitKind::DeliberateSeverance;
11669    }
11670    exit_report
11671}
11672
11673fn classify_reaped_child_exit(
11674    snapshot: &SharedSnapshot,
11675    child: &SupervisedChild,
11676    status: &ExitStatus,
11677) -> ExitReport {
11678    let _ = update_snapshot(snapshot, None, |state| {
11679        state.reaped_pid = Some(child.pid);
11680        state.spawn_failure = child.spawn_failure.clone();
11681    });
11682    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11683}
11684
11685fn fail_snapshot(
11686    snapshot: &SharedSnapshot,
11687    module_id: Option<&str>,
11688    last_exit: Option<ExitReport>,
11689    failure: &FailureContext,
11690) {
11691    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11692        state.state = ModuleState::Failed;
11693        clear_current_process_facts(state);
11694        if failure.may_retry() && state.enabled {
11695            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11696        } else if failure.shutdown.is_set() {
11697            state.failed_retry.cancel();
11698        }
11699        if let Some(last_exit) = last_exit {
11700            state.last_exit = Some(last_exit);
11701        }
11702    }) {
11703        error!(error = %err, "failed to mark supervisor state failed");
11704    }
11705}
11706
11707fn update_snapshot(
11708    snapshot: &SharedSnapshot,
11709    module_id: Option<&str>,
11710    update: impl FnOnce(&mut SupervisorSnapshot),
11711) -> Result<(), SuperviseError> {
11712    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11713        module_id: module_id.map(ToOwned::to_owned),
11714    })?;
11715    update(&mut state);
11716    Ok(())
11717}
11718
11719const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11720
11721fn lock_snapshot_for_control<'a>(
11722    snapshot: &'a SharedSnapshot,
11723    module_id: &str,
11724    caller: &'static str,
11725) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11726    let started_at = Instant::now();
11727    let guard = lock_snapshot(snapshot)?;
11728    let waited = started_at.elapsed();
11729    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11730        warn!(
11731            module_id = %module_id,
11732            waited_ms = waited.as_millis() as u64,
11733            caller = %caller,
11734            "slow snapshot lock"
11735        );
11736    }
11737    Ok(guard)
11738}
11739
11740fn lock_snapshot(
11741    snapshot: &SharedSnapshot,
11742) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11743    snapshot
11744        .lock()
11745        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11746}
11747
11748#[cfg(test)]
11749mod terminal_history_tests {
11750    use std::{
11751        path::PathBuf,
11752        sync::Arc,
11753        time::{Duration, Instant},
11754    };
11755
11756    use tokio::{sync::mpsc, time::sleep};
11757
11758    use super::{
11759        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11760        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11761        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11762        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11763        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11764        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11765        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11766        SupervisorHealthStatus, SupervisorSnapshot,
11767    };
11768    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11769    // use for their own wall-clock deadlines: crash-restart instants must be on
11770    // the same clock the production code stamps them with, which is tokio's (and
11771    // is what `start_paused` tests can move).
11772    use super::Instant as ClockInstant;
11773    use crate::{
11774        registry::Registry,
11775        terminal_ring::{TerminalRing, TerminalRingConfig},
11776    };
11777    use std::sync::Mutex;
11778    use subc_control::TerminalDisposition;
11779
11780    /// See the twin in `control.rs` for why this derives the path from
11781    /// `current_exe()` and why the existence check is here: `--lib` alone does
11782    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11783    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11784    pub(super) fn fake_aft_stub_path() -> PathBuf {
11785        let mut path = std::env::current_exe().expect("current_exe available in tests");
11786        path.pop();
11787        path.pop();
11788        path.push(if cfg!(windows) {
11789            "fake-aft-stub.exe"
11790        } else {
11791            "fake-aft-stub"
11792        });
11793        assert!(
11794            path.exists(),
11795            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11796             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11797            path.display()
11798        );
11799        path
11800    }
11801
11802    #[test]
11803    fn reserved_never_spawned_refuses_every_hello() {
11804        // The canary hole: a reserved id whose module has never spawned had NO
11805        // gate entry and admitted anyone -- the reservation protected the nonce
11806        // holder, not the NAME. Now the entry is present with no legitimate
11807        // holder and refuses all comers.
11808        let supervisor = SupervisorHandle::default();
11809        supervisor.apply_identity_configuration(&ModuleSpec {
11810            module_id: "never-spawned".to_string(),
11811            program: PathBuf::from("/usr/bin/false"),
11812            args: Vec::new(),
11813            env: Vec::new(),
11814            reserved: true,
11815            reserved_prefixes: Vec::new(),
11816            protocol: ModuleProtocol::Subc,
11817            overlap: Default::default(),
11818        });
11819        assert!(
11820            supervisor
11821                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11822                .is_err(),
11823            "forged nonce must refuse on a reserved never-spawned id"
11824        );
11825        assert!(
11826            supervisor
11827                .reserved_hello_verification("never-spawned", None)
11828                .is_err(),
11829            "absent nonce must refuse on a reserved never-spawned id"
11830        );
11831        // And a real spawn nonce minted later admits exactly that nonce.
11832        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11833        supervisor.apply_identity_configuration(&ModuleSpec {
11834            module_id: "never-spawned".to_string(),
11835            program: PathBuf::from("/usr/bin/false"),
11836            args: Vec::new(),
11837            env: Vec::new(),
11838            reserved: true,
11839            reserved_prefixes: Vec::new(),
11840            protocol: ModuleProtocol::Subc,
11841            overlap: Default::default(),
11842        });
11843        assert!(supervisor
11844            .reserved_hello_verification("never-spawned", Some("minted"))
11845            .is_ok());
11846        assert!(supervisor
11847            .reserved_hello_verification("never-spawned", Some("forged"))
11848            .is_err());
11849    }
11850
11851    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11852    /// happened, which is what "spent budget" looks like to every reader.
11853    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11854        let now = ClockInstant::now();
11855        for _ in 0..count {
11856            state.crash_restarts.push_back(now);
11857        }
11858    }
11859
11860    /// Age the oldest recorded restart out of `window`, standing in for the hours
11861    /// that would otherwise have to pass. Injecting the instant is the point: a
11862    /// test that slept a real window would take ten minutes and still prove less.
11863    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11864        let aged = state
11865            .crash_restarts
11866            .front()
11867            .expect("a crash restart must be recorded before it can be aged")
11868            .checked_sub(window + Duration::from_secs(1))
11869            .expect("the test clock is far enough from its origin to age an instant");
11870        state.crash_restarts[0] = aged;
11871    }
11872
11873    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11874        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11875        seed_crash_restarts(&mut state, count);
11876        state
11877    }
11878
11879    #[test]
11880    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11881        let policy = RestartPolicy::new(3, Duration::ZERO);
11882        let now = ClockInstant::now();
11883        assert!(daemon_will_restart(
11884            &mut snapshot_with_restarts(true, 2),
11885            &policy,
11886            now
11887        ));
11888        assert!(!daemon_will_restart(
11889            &mut snapshot_with_restarts(true, 3),
11890            &policy,
11891            now
11892        ));
11893        assert!(!daemon_will_restart(
11894            &mut snapshot_with_restarts(false, 0),
11895            &policy,
11896            now
11897        ));
11898    }
11899
11900    #[test]
11901    fn crash_restart_backoff_escalates_with_in_window_count() {
11902        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11903            .with_max_backoff(Duration::from_secs(30));
11904        let now = ClockInstant::now();
11905        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11906        let schedules = (0..4)
11907            .map(|_| {
11908                state
11909                    .next_crash_restart(&policy, now)
11910                    .expect("the test policy allows four crash restarts")
11911            })
11912            .collect::<Vec<_>>();
11913
11914        assert_eq!(
11915            schedules
11916                .iter()
11917                .map(|schedule| schedule.restart_in_window)
11918                .collect::<Vec<_>>(),
11919            vec![0, 1, 2, 3]
11920        );
11921        assert_eq!(
11922            schedules
11923                .iter()
11924                .map(|schedule| schedule.delay)
11925                .collect::<Vec<_>>(),
11926            vec![
11927                Duration::from_millis(100),
11928                Duration::from_secs(1),
11929                Duration::from_secs(10),
11930                Duration::from_secs(30),
11931            ]
11932        );
11933    }
11934
11935    #[test]
11936    fn crash_restart_backoff_resets_after_ring_clear() {
11937        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11938        let now = ClockInstant::now();
11939        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11940        assert_eq!(
11941            state.next_crash_restart(&policy, now).unwrap().delay,
11942            Duration::from_millis(100)
11943        );
11944        assert_eq!(
11945            state.next_crash_restart(&policy, now).unwrap().delay,
11946            Duration::from_secs(1)
11947        );
11948
11949        state.clear_crash_restarts();
11950        let schedule = state
11951            .next_crash_restart(&policy, now)
11952            .expect("a cleared ring must allow another restart");
11953        assert_eq!(schedule.restart_in_window, 0);
11954        assert_eq!(schedule.delay, Duration::from_millis(100));
11955    }
11956
11957    #[test]
11958    fn crash_restart_backoff_ignores_aged_restarts() {
11959        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11960        let now = ClockInstant::now();
11961        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11962        state
11963            .next_crash_restart(&policy, now)
11964            .expect("the first restart is allowed");
11965        state
11966            .next_crash_restart(&policy, now)
11967            .expect("the second restart is allowed");
11968        state.crash_restarts[0] = now
11969            .checked_sub(policy.window + Duration::from_secs(1))
11970            .expect("the fake clock can age a restart past the window");
11971
11972        let schedule = state
11973            .next_crash_restart(&policy, now)
11974            .expect("an aged restart must release its slot");
11975        assert_eq!(schedule.restart_in_window, 1);
11976        assert_eq!(schedule.delay, Duration::from_secs(1));
11977        assert_eq!(state.crash_restarts.len(), 2);
11978    }
11979
11980    /// The budget is a rate: the same three spent restarts refuse a respawn
11981    /// while they are recent and allow one once they have aged past the window.
11982    /// Nothing about the module changed in between, which is the whole point.
11983    #[test]
11984    fn a_budget_spent_before_the_window_no_longer_refuses() {
11985        let policy = RestartPolicy::new(3, Duration::ZERO);
11986        let mut state = snapshot_with_restarts(true, 3);
11987        let now = ClockInstant::now();
11988        assert!(!daemon_will_restart(&mut state, &policy, now));
11989
11990        assert!(daemon_will_restart(
11991            &mut state,
11992            &policy,
11993            now + policy.window + Duration::from_secs(1)
11994        ));
11995        assert!(
11996            state.crash_restarts.is_empty(),
11997            "reading the budget must drop the instants that left the window"
11998        );
11999    }
12000
12001    fn module_with_recovery_snapshot(
12002        state: ModuleState,
12003        enabled: bool,
12004        restart_count: u32,
12005    ) -> SupervisedModule {
12006        let registry = Arc::new(Registry::default());
12007        let supervisor =
12008            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12009        let runtime = supervisor.runtime_config();
12010        let spec = ModuleSpec {
12011            module_id: "recovery-snapshot".to_string(),
12012            program: fake_aft_stub_path(),
12013            args: Vec::new(),
12014            env: Vec::new(),
12015            reserved: false,
12016            reserved_prefixes: Vec::new(),
12017            protocol: ModuleProtocol::Subc,
12018            overlap: Default::default(),
12019        };
12020        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12021        seed_crash_restarts(&mut snapshot, restart_count);
12022        // These tests read synthetic snapshots. A real child and monitor would
12023        // race those reads by replacing the requested state during startup.
12024        let (commands, _rx) = mpsc::channel(4);
12025        SupervisedModule {
12026            inner: Arc::new(SupervisedModuleInner {
12027                module_id: spec.module_id.clone(),
12028                registry,
12029                snapshot: Arc::new(Mutex::new(snapshot)),
12030                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12031                    spec,
12032                    health: runtime.health,
12033                })),
12034                stderr_ring: runtime.stderr_ring,
12035                terminal_ring: runtime.terminal_ring,
12036                commands,
12037                monitor: Mutex::new(None),
12038                restart_policy: runtime.restart_policy,
12039                effective_drain_timeout: runtime.effective_drain_timeout,
12040                provenance_probe: supervisor.provenance_probe.clone(),
12041            }),
12042        }
12043    }
12044
12045    #[cfg(target_os = "linux")]
12046    #[tokio::test]
12047    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12048        let supervisor =
12049            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12050                .with_cgroup_placement(None);
12051        let result = supervisor.spawn(ModuleSpec {
12052            module_id: "no-cgroup-placement".to_string(),
12053            program: fake_aft_stub_path(),
12054            args: Vec::new(),
12055            env: Vec::new(),
12056            reserved: false,
12057            reserved_prefixes: Vec::new(),
12058            protocol: ModuleProtocol::Subc,
12059            overlap: Default::default(),
12060        });
12061
12062        assert!(
12063            result.is_ok(),
12064            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12065        );
12066    }
12067
12068    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12069    async fn undecided_snapshot_uses_shared_restart_predicate() {
12070        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12071            .will_recover_after_connection_loss()
12072            .unwrap());
12073        assert!(
12074            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12075                .will_recover_after_connection_loss()
12076                .unwrap()
12077        );
12078    }
12079
12080    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12081    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12082        assert!(
12083            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12084                .will_recover_after_connection_loss()
12085                .unwrap()
12086        );
12087    }
12088
12089    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12090    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12091        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12092            .will_recover_after_connection_loss()
12093            .unwrap());
12094        assert!(
12095            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12096                .will_recover_after_connection_loss()
12097                .unwrap()
12098        );
12099    }
12100
12101    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12102    async fn warming_snapshot_is_limited_to_startup_phases() {
12103        for state in [
12104            ModuleState::Starting,
12105            ModuleState::Running,
12106            ModuleState::Restarting,
12107        ] {
12108            assert!(
12109                module_with_recovery_snapshot(state, true, 0)
12110                    .is_warming()
12111                    .unwrap(),
12112                "{state:?} should be warming"
12113            );
12114        }
12115        for state in [
12116            ModuleState::Unresponsive,
12117            ModuleState::Draining,
12118            ModuleState::Stopped,
12119            ModuleState::Failed,
12120            ModuleState::Disabled,
12121        ] {
12122            assert!(
12123                !module_with_recovery_snapshot(state, true, 0)
12124                    .is_warming()
12125                    .unwrap(),
12126                "{state:?} should not be warming"
12127            );
12128        }
12129    }
12130
12131    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12132    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12133        let registry = Arc::new(Registry::default());
12134        let supervisor =
12135            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12136        let module = supervisor
12137            .spawn(ModuleSpec {
12138                module_id: "terminal-history".to_string(),
12139                program: fake_aft_stub_path(),
12140                args: Vec::new(),
12141                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12142                reserved: false,
12143                reserved_prefixes: Vec::new(),
12144                protocol: ModuleProtocol::Subc,
12145                overlap: Default::default(),
12146            })
12147            .unwrap();
12148
12149        let deadline = Instant::now() + Duration::from_secs(5);
12150        loop {
12151            let history = module.terminal_history();
12152            if history.entries.len() == 2 {
12153                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12154                assert_eq!(history.dropped, 0);
12155                assert_eq!(
12156                    history
12157                        .entries
12158                        .iter()
12159                        .map(|entry| entry.exit_code)
12160                        .collect::<Vec<_>>(),
12161                    vec![Some(23), Some(23)]
12162                );
12163                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12164                return;
12165            }
12166            assert!(
12167                Instant::now() < deadline,
12168                "module did not retain two terminal exits: {history:?}"
12169            );
12170            sleep(Duration::from_millis(10)).await;
12171        }
12172    }
12173
12174    /// A disable issued while a crash respawn is still backing off must preempt
12175    /// that respawn: the operator's stop wins, the disable must not queue behind
12176    /// the backoff, and the module must never come back up afterwards.
12177    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12178    async fn disable_during_crash_backoff_cancels_pending_respawn() {
12179        let backoff = Duration::from_secs(2);
12180        let supervisor = Supervisor::new_for_test(
12181            Arc::new(Registry::default()),
12182            RestartPolicy::new(10, backoff),
12183        );
12184        let module = supervisor
12185            .spawn(ModuleSpec {
12186                module_id: "disable-during-backoff".to_string(),
12187                program: fake_aft_stub_path(),
12188                args: Vec::new(),
12189                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12190                reserved: false,
12191                reserved_prefixes: Vec::new(),
12192                protocol: ModuleProtocol::Subc,
12193                overlap: Default::default(),
12194            })
12195            .unwrap();
12196
12197        // Wait for the first crash to put the module into its backoff window.
12198        let deadline = Instant::now() + Duration::from_secs(5);
12199        loop {
12200            if module.status().unwrap().state == ModuleState::Restarting {
12201                break;
12202            }
12203            assert!(
12204                Instant::now() < deadline,
12205                "module never entered the crash backoff"
12206            );
12207            sleep(Duration::from_millis(10)).await;
12208        }
12209
12210        let started = Instant::now();
12211        module.set_enabled(false).await.unwrap();
12212        let waited = started.elapsed();
12213
12214        assert!(
12215            waited < backoff / 2,
12216            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12217        );
12218        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12219
12220        // Outlast the backoff: the respawn it was counting down to must never run.
12221        sleep(backoff + Duration::from_millis(500)).await;
12222        let status = module.status().unwrap();
12223        assert_eq!(status.state, ModuleState::Disabled);
12224        assert_eq!(
12225            status.spawn_generation, 1,
12226            "module respawned after the operator disabled it"
12227        );
12228    }
12229
12230    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12231    /// the shape of nats-server, the program this rule exists for.
12232    #[cfg(unix)]
12233    fn protocol_none_sigterm_exits_clean_spec(
12234        module_id: &str,
12235        dir: &std::path::Path,
12236    ) -> (ModuleSpec, PathBuf, PathBuf) {
12237        let ready = dir.join("ready");
12238        let marker = dir.join("sigterm");
12239        let spec = ModuleSpec {
12240            module_id: module_id.to_string(),
12241            program: fake_aft_stub_path(),
12242            args: Vec::new(),
12243            env: vec![
12244                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12245                (
12246                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12247                    marker.display().to_string(),
12248                ),
12249                (
12250                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12251                    ready.display().to_string(),
12252                ),
12253            ],
12254            reserved: false,
12255            reserved_prefixes: Vec::new(),
12256            protocol: ModuleProtocol::None,
12257            overlap: Default::default(),
12258        };
12259        (spec, ready, marker)
12260    }
12261
12262    /// Wait for a file the child writes, so a signal is never sent before the
12263    /// child's SIGTERM handler is installed (the default disposition would
12264    /// kill it by signal and the exit would not be clean).
12265    #[cfg(unix)]
12266    async fn wait_for_file(path: &std::path::Path) {
12267        let deadline = Instant::now() + Duration::from_secs(10);
12268        while !path.exists() {
12269            assert!(
12270                Instant::now() < deadline,
12271                "{} never appeared",
12272                path.display()
12273            );
12274            sleep(Duration::from_millis(10)).await;
12275        }
12276    }
12277
12278    /// A protocol-none module that exits 0 because something OUTSIDE the
12279    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12280    /// the crash-path disposition rather than `stopped`.
12281    #[cfg(unix)]
12282    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12283    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12284        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12285        let (spec, ready, marker) =
12286            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12287        let supervisor = Supervisor::new_for_test(
12288            Arc::new(Registry::default()),
12289            RestartPolicy::new(3, Duration::ZERO),
12290        );
12291        let module = supervisor.spawn(spec).unwrap();
12292        wait_for_file(&ready).await;
12293        // The ready file proves the child installed its SIGTERM handler, not
12294        // that the supervisor has processed the privacy trampoline's exec
12295        // acknowledgement. On macOS status withholds the pid until then.
12296        let deadline = Instant::now() + Duration::from_secs(10);
12297        let first_pid = loop {
12298            let status = module.status().unwrap();
12299            if status.state == ModuleState::Running {
12300                if let Some(pid) = status.pid {
12301                    break pid;
12302                }
12303            }
12304            assert!(
12305                Instant::now() < deadline,
12306                "a running module must report its pid after exec confirmation: {status:?}"
12307            );
12308            sleep(Duration::from_millis(10)).await;
12309        };
12310
12311        rustix::process::kill_process(
12312            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12313            rustix::process::Signal::TERM,
12314        )
12315        .unwrap();
12316
12317        let deadline = Instant::now() + Duration::from_secs(10);
12318        let respawned = loop {
12319            let status = module.status().unwrap();
12320            if status.state == ModuleState::Running
12321                && status.pid.is_some_and(|pid| pid != first_pid)
12322            {
12323                break status;
12324            }
12325            assert!(
12326                Instant::now() < deadline,
12327                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12328            );
12329            sleep(Duration::from_millis(10)).await;
12330        };
12331        assert_eq!(respawned.spawn_generation, 2);
12332        assert!(
12333            marker.exists(),
12334            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12335        );
12336
12337        let history = module.terminal_history();
12338        assert_eq!(history.entries.len(), 1, "{history:?}");
12339        let entry = &history.entries[0];
12340        assert_eq!(entry.exit_code, Some(0));
12341        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12342        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12343
12344        module.stop().await.unwrap();
12345    }
12346
12347    /// Repeated unrequested clean exits of a protocol-none module spend the
12348    /// restart budget exactly as crashes do, and the module ends `failed` with
12349    /// the budget named.
12350    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12351    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12352        let supervisor = Supervisor::new_for_test(
12353            Arc::new(Registry::default()),
12354            RestartPolicy::new(1, Duration::ZERO),
12355        );
12356        let module = supervisor
12357            .spawn(ModuleSpec {
12358                module_id: "none-clean-exit-budget".to_string(),
12359                program: fake_aft_stub_path(),
12360                args: Vec::new(),
12361                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12362                reserved: false,
12363                reserved_prefixes: Vec::new(),
12364                protocol: ModuleProtocol::None,
12365                overlap: Default::default(),
12366            })
12367            .unwrap();
12368
12369        // Failed follows the terminal write; this deadline only bounds a hang,
12370        // not an assumed duration for the two launches or their exit recording.
12371        let deadline = Instant::now() + Duration::from_secs(10);
12372        loop {
12373            let status = module.status().unwrap();
12374            if status.state == ModuleState::Failed {
12375                break;
12376            }
12377            assert!(
12378                Instant::now() < deadline,
12379                "module never exhausted its budget: {status:?} {:?}",
12380                module.terminal_history()
12381            );
12382            sleep(Duration::from_millis(10)).await;
12383        }
12384        let history = module.terminal_history();
12385        assert_eq!(
12386            history
12387                .entries
12388                .iter()
12389                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12390                .collect::<Vec<_>>(),
12391            vec![
12392                (Some(0), TerminalDisposition::Restarting),
12393                (Some(0), TerminalDisposition::Failed),
12394            ]
12395        );
12396        let detail = history.entries[1]
12397            .disposition_detail
12398            .as_deref()
12399            .expect("a budget failure names the budget");
12400        assert!(detail.contains("max_restarts=1"), "{detail}");
12401        assert_eq!(module.status().unwrap().spawn_generation, 2);
12402    }
12403
12404    #[test]
12405    fn restart_budget_failure_is_published_after_its_terminal_record() {
12406        use std::io::{Read, Write};
12407        let supervisor = Supervisor::new_for_test(
12408            Arc::new(Registry::default()),
12409            RestartPolicy::new(0, Duration::ZERO),
12410        );
12411        let runtime = supervisor.runtime_config();
12412        let spec = ModuleSpec {
12413            protocol: ModuleProtocol::None,
12414            ..windowed_crash_spec("budget-publication-order")
12415        };
12416        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12417            ModuleState::Running,
12418            true,
12419        )));
12420        lock_snapshot(&snapshot).unwrap().process_alive = true;
12421        let reaped_snapshot = snapshot.clone();
12422        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12423        let history = runtime.terminal_ring.clone();
12424        let reap = std::thread::spawn(move || {
12425            tokio::runtime::Builder::new_current_thread()
12426                .enable_all()
12427                .build()
12428                .unwrap()
12429                .block_on(on_child_exit(
12430                    &spec,
12431                    runtime.restart_policy,
12432                    &supervisor.registry,
12433                    &reaped_snapshot,
12434                    &runtime.terminal_ring,
12435                    &runtime.spawn_events,
12436                    &runtime.child_roster,
12437                    ExitReport {
12438                        kind: ExitKind::Clean,
12439                        code: Some(0),
12440                        signal: None,
12441                        at_ms: 1,
12442                    },
12443                    true,
12444                ))
12445        });
12446        let mut reached = [0];
12447        barrier.read_exact(&mut reached).unwrap();
12448        assert_eq!(&reached, b"R");
12449        let (before_state, before_alive) = {
12450            let state = lock_snapshot(&snapshot).unwrap();
12451            (state.state, state.process_alive)
12452        };
12453        let before_history = history.lock().unwrap().snapshot();
12454        barrier.write_all(b"X").unwrap();
12455        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12456        assert!(before_history.entries.is_empty());
12457        assert_ne!(
12458            before_state,
12459            ModuleState::Failed,
12460            "Failed was visible before its terminal record could be written"
12461        );
12462        assert!(before_alive);
12463        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12464        let history = history.lock().unwrap().snapshot();
12465        assert_eq!(history.entries.len(), 1);
12466        assert_eq!(history.entries[0].exit_code, Some(0));
12467        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12468    }
12469
12470    #[test]
12471    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12472        use std::io::{Read, Write};
12473
12474        let supervisor = Supervisor::new_for_test(
12475            Arc::new(Registry::default()),
12476            RestartPolicy::new(3, Duration::ZERO),
12477        );
12478        let mut runtime = supervisor.runtime_config();
12479        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12480        let snapshot = module.inner.snapshot.clone();
12481        lock_snapshot(&snapshot).unwrap().process_alive = true;
12482        runtime.terminal_ring = module.inner.terminal_ring.clone();
12483        let spec = windowed_crash_spec(module.module_id());
12484        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12485        let reaped_snapshot = snapshot.clone();
12486        let reap = std::thread::spawn(move || {
12487            tokio::runtime::Builder::new_current_thread()
12488                .enable_all()
12489                .build()
12490                .unwrap()
12491                .block_on(on_child_exit(
12492                    &spec,
12493                    runtime.restart_policy,
12494                    &supervisor.registry,
12495                    &reaped_snapshot,
12496                    &runtime.terminal_ring,
12497                    &runtime.spawn_events,
12498                    &runtime.child_roster,
12499                    ExitReport {
12500                        kind: ExitKind::Crash,
12501                        code: Some(2),
12502                        signal: None,
12503                        at_ms: 1,
12504                    },
12505                    true,
12506                ))
12507        });
12508
12509        // Hold the crash handler between its budget decision and publication.
12510        // Connection cleanup must still see the recovery that this crash chose,
12511        // even when that recovery consumes the last available restart slot.
12512        let mut reached = [0];
12513        barrier.read_exact(&mut reached).unwrap();
12514        assert_eq!(&reached, b"R");
12515        let before_status = module.status().unwrap();
12516        let before_history = module.terminal_history();
12517        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12518        barrier.write_all(b"X").unwrap();
12519        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12520
12521        assert_eq!(before_status.state, ModuleState::Running);
12522        assert!(before_status.process_alive);
12523        assert_eq!(before_status.restart_count, 3);
12524        assert!(before_history.entries.is_empty());
12525        assert!(
12526            before_recovers,
12527            "connection cleanup would report a terminal close for a scheduled crash restart"
12528        );
12529        let restarting = module.status().unwrap();
12530        assert_eq!(restarting.state, ModuleState::Restarting);
12531        assert_eq!(restarting.restart_count, 3);
12532        assert!(module.will_recover_after_connection_loss().unwrap());
12533        let history = module.terminal_history();
12534        assert_eq!(history.entries.len(), 1);
12535        assert_eq!(
12536            history.entries[0].disposition,
12537            TerminalDisposition::Restarting
12538        );
12539
12540        // Once the replacement is running, its next crash must be judged
12541        // against the exhausted budget rather than the previous recovery.
12542        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12543            state.state = ModuleState::Running;
12544            state.respawn_pending = false;
12545        })
12546        .unwrap();
12547        assert!(!module.will_recover_after_connection_loss().unwrap());
12548        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12549            state.enabled = false;
12550            state.respawn_pending = true;
12551        })
12552        .unwrap();
12553        assert!(!module.will_recover_after_connection_loss().unwrap());
12554    }
12555
12556    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12557        use std::net::{TcpListener, TcpStream};
12558        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12559        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12560        let (barrier, _) = listener.accept().unwrap();
12561        for stream in [&writer, &barrier] {
12562            stream
12563                .set_read_timeout(Some(Duration::from_secs(10)))
12564                .unwrap();
12565            stream
12566                .set_write_timeout(Some(Duration::from_secs(10)))
12567                .unwrap();
12568        }
12569        *events.1.lock().unwrap() = Some(writer);
12570        barrier
12571    }
12572
12573    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12574        use std::io::{Read, Write};
12575        let barrier = events.1.lock().unwrap().take();
12576        if let Some(mut barrier) = barrier {
12577            barrier.write_all(b"R").unwrap();
12578            let mut release = [0];
12579            barrier.read_exact(&mut release).unwrap();
12580            assert_eq!(&release, b"X");
12581        }
12582    }
12583
12584    #[test]
12585    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12586        use std::io::{Read, Write};
12587
12588        let supervisor = Supervisor::new_for_test(
12589            Arc::new(Registry::default()),
12590            RestartPolicy::new(0, Duration::ZERO),
12591        );
12592        let mut runtime = supervisor.runtime_config();
12593        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12594        let snapshot = module.inner.snapshot.clone();
12595        runtime.terminal_ring = module.inner.terminal_ring.clone();
12596        let spec = ModuleSpec {
12597            module_id: module.module_id().into(),
12598            program: fake_aft_stub_path(),
12599            protocol: ModuleProtocol::None,
12600            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12601            ..windowed_crash_spec(module.module_id())
12602        };
12603        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12604        let reap = std::thread::spawn(move || {
12605            tokio::runtime::Builder::new_current_thread()
12606                .enable_all()
12607                .build()
12608                .unwrap()
12609                .block_on(async {
12610                    let mut child =
12611                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12612                    health_restart_child(
12613                        &spec,
12614                        &runtime,
12615                        &supervisor.registry,
12616                        &supervisor.process_liveness,
12617                        &snapshot,
12618                        &mut child,
12619                        SupervisorHealthStatus::Failing,
12620                        Some("health probe did not answer"),
12621                        1,
12622                    )
12623                    .await
12624                    .unwrap();
12625                    assert!(child.is_none());
12626                });
12627        });
12628        // The loopback barrier acknowledges entry into terminal recording by
12629        // health_restart_child's drain, then waits for the test's release byte.
12630        // Status and history are observed while that writer is held, not after
12631        // an elapsed-time guess about when the health action has completed.
12632        let mut reached = [0];
12633        barrier.read_exact(&mut reached).unwrap();
12634        assert_eq!(&reached, b"R");
12635        let before_status = module.status().unwrap();
12636        let before_history = module.terminal_history();
12637        barrier.write_all(b"X").unwrap();
12638        reap.join().unwrap();
12639        assert!(before_history.entries.is_empty());
12640        assert_ne!(
12641            before_status.state,
12642            ModuleState::Failed,
12643            "Failed was visible before its terminal record could be written"
12644        );
12645        assert!(
12646            before_status.process_alive,
12647            "process death was visible before its terminal record could be written"
12648        );
12649        let failed = module.status().unwrap();
12650        assert_eq!(failed.state, ModuleState::Failed);
12651        assert!(!failed.process_alive);
12652        let history = module.terminal_history();
12653        assert_eq!(history.entries.len(), 1);
12654        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12655        assert_eq!(
12656            history.entries[0].disposition_detail.as_deref(),
12657            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12658        );
12659    }
12660
12661    #[test]
12662    fn observed_exits_publish_process_death_after_terminal_history() {
12663        use std::io::{Read, Write};
12664        for (kind, enabled, shutdown, wait_error, expected_state) in [
12665            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12666            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12667            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12668            (
12669                ExitKind::DeliberateSeverance,
12670                true,
12671                false,
12672                false,
12673                ModuleState::Restarting,
12674            ),
12675            #[cfg(unix)]
12676            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12677            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12678        ] {
12679            let supervisor = Supervisor::new_for_test(
12680                Arc::new(Registry::default()),
12681                RestartPolicy::new(1, Duration::ZERO),
12682            );
12683            let runtime = supervisor.runtime_config();
12684            #[cfg(unix)]
12685            if shutdown {
12686                runtime.child_roster.close();
12687            }
12688            #[cfg(not(unix))]
12689            let _ = shutdown;
12690            let spec = windowed_crash_spec("exit-publication-order");
12691            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12692                ModuleState::Running,
12693                enabled,
12694            )));
12695            lock_snapshot(&snapshot).unwrap().process_alive = true;
12696            let reaped_snapshot = snapshot.clone();
12697            let history = runtime.terminal_ring.clone();
12698            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12699            let reap = std::thread::spawn(move || {
12700                if wait_error {
12701                    super::fail_after_wait_error(
12702                        &spec.module_id,
12703                        &reaped_snapshot,
12704                        &runtime.terminal_ring,
12705                        &runtime.spawn_events,
12706                        "wait failed".into(),
12707                        &runtime.failure_context(true),
12708                    );
12709                } else {
12710                    tokio::runtime::Builder::new_current_thread()
12711                        .enable_all()
12712                        .build()
12713                        .unwrap()
12714                        .block_on(on_child_exit(
12715                            &spec,
12716                            runtime.restart_policy,
12717                            &supervisor.registry,
12718                            &reaped_snapshot,
12719                            &runtime.terminal_ring,
12720                            &runtime.spawn_events,
12721                            &runtime.child_roster,
12722                            ExitReport {
12723                                kind,
12724                                code: Some(0),
12725                                signal: None,
12726                                at_ms: 1,
12727                            },
12728                            true,
12729                        ));
12730                }
12731            });
12732            let mut reached = [0];
12733            barrier.read_exact(&mut reached).unwrap();
12734            assert_eq!(&reached, b"R");
12735            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12736            let before_history = history.lock().unwrap().snapshot();
12737            barrier.write_all(b"X").unwrap();
12738            reap.join().unwrap();
12739            assert!(
12740                before_alive,
12741                "{kind:?}: death was published before recording"
12742            );
12743            assert!(before_history.entries.is_empty());
12744            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12745            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12746            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12747        }
12748    }
12749
12750    #[tokio::test]
12751    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12752        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12753        let path = dir.join("terminals.jsonl");
12754        std::fs::create_dir(&path).unwrap();
12755        let supervisor = Supervisor::new_for_test(
12756            Arc::new(Registry::default()),
12757            RestartPolicy::new(0, Duration::ZERO),
12758        )
12759        .with_terminal_journal(path, "health-budget-journal-failure".into());
12760        let runtime = supervisor.runtime_config();
12761        let spec = ModuleSpec {
12762            program: fake_aft_stub_path(),
12763            protocol: ModuleProtocol::None,
12764            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12765            ..windowed_crash_spec("health-budget-journal-failure")
12766        };
12767        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12768        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12769        health_restart_child(
12770            &spec,
12771            &runtime,
12772            &supervisor.registry,
12773            &supervisor.process_liveness,
12774            &snapshot,
12775            &mut child,
12776            SupervisorHealthStatus::Failing,
12777            None,
12778            1,
12779        )
12780        .await
12781        .unwrap();
12782        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12783        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12784        let history = runtime
12785            .terminal_ring
12786            .lock()
12787            .unwrap()
12788            .durable_history(&spec.module_id);
12789        assert!(history.journal_write_failures > 0);
12790        assert_eq!(history.entries.len(), 1);
12791        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12792    }
12793
12794    #[test]
12795    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12796        let supervisor = Supervisor::new_for_test(
12797            Arc::new(Registry::default()),
12798            RestartPolicy::new(0, Duration::ZERO),
12799        );
12800        let runtime = supervisor.runtime_config();
12801        let spec = ModuleSpec {
12802            program: fake_aft_stub_path(),
12803            protocol: ModuleProtocol::None,
12804            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12805            ..windowed_crash_spec("health-budget-recording-panic")
12806        };
12807        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12808        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12809            tokio::runtime::Builder::new_current_thread()
12810                .enable_all()
12811                .build()
12812                .unwrap()
12813                .block_on(async {
12814                    let mut child =
12815                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12816                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12817                    health_restart_child(
12818                        &spec,
12819                        &runtime,
12820                        &supervisor.registry,
12821                        &supervisor.process_liveness,
12822                        &snapshot,
12823                        &mut child,
12824                        SupervisorHealthStatus::Failing,
12825                        None,
12826                        1,
12827                    )
12828                    .await
12829                    .unwrap();
12830                });
12831        }));
12832        let panic = result.expect_err("recording must still unwind");
12833        assert_eq!(
12834            panic.downcast_ref::<String>().map(String::as_str),
12835            Some("spawn event sequence exhausted")
12836        );
12837        let state = lock_snapshot(&snapshot).unwrap();
12838        assert_eq!(state.state, ModuleState::Failed);
12839        assert!(!state.process_alive);
12840        assert!(state.last_exit.is_some());
12841    }
12842
12843    #[tokio::test]
12844    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12845        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12846        let path = dir.join("terminals.jsonl");
12847        std::fs::create_dir(&path).unwrap();
12848        let supervisor = Supervisor::new_for_test(
12849            Arc::new(Registry::default()),
12850            RestartPolicy::new(0, Duration::ZERO),
12851        )
12852        .with_terminal_journal(path, "budget-journal-failure".into());
12853        let runtime = supervisor.runtime_config();
12854        let spec = windowed_crash_spec("budget-journal-failure");
12855        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12856        assert!(matches!(
12857            on_child_exit(
12858                &spec,
12859                runtime.restart_policy,
12860                &supervisor.registry,
12861                &snapshot,
12862                &runtime.terminal_ring,
12863                &runtime.spawn_events,
12864                &runtime.child_roster,
12865                crash_exit_report(1),
12866                true,
12867            )
12868            .await,
12869            NextAction::Stop { .. }
12870        ));
12871        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12872        let history = runtime
12873            .terminal_ring
12874            .lock()
12875            .unwrap()
12876            .durable_history(&spec.module_id);
12877        assert!(history.journal_write_failures > 0);
12878        assert_eq!(history.entries.len(), 1);
12879        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12880    }
12881
12882    #[test]
12883    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12884        let supervisor = Supervisor::new_for_test(
12885            Arc::new(Registry::default()),
12886            RestartPolicy::new(0, Duration::ZERO),
12887        );
12888        let runtime = supervisor.runtime_config();
12889        let spec = windowed_crash_spec("budget-recording-panic");
12890        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12891        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12892        // Exhausting the event sequence makes emit_exited panic before the
12893        // terminal write, exercising publication on the recording unwind.
12894        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12895        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12896            tokio::runtime::Builder::new_current_thread()
12897                .enable_all()
12898                .build()
12899                .unwrap()
12900                .block_on(on_child_exit(
12901                    &spec,
12902                    runtime.restart_policy,
12903                    &supervisor.registry,
12904                    &snapshot,
12905                    &runtime.terminal_ring,
12906                    &runtime.spawn_events,
12907                    &runtime.child_roster,
12908                    crash_exit_report(1),
12909                    true,
12910                ))
12911        }));
12912        let panic = result.err().expect("recording must still unwind");
12913        assert_eq!(
12914            panic.downcast_ref::<String>().map(String::as_str),
12915            Some("spawn event sequence exhausted")
12916        );
12917        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12918    }
12919
12920    /// A stop the supervisor itself requests still stops a protocol-none
12921    /// module, even though the child answers the SIGTERM with exit 0.
12922    #[cfg(unix)]
12923    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12924    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12925        for disable in [false, true] {
12926            let label = if disable {
12927                "none-requested-disable"
12928            } else {
12929                "none-requested-stop"
12930            };
12931            let dir = cortexkit_test_support::ScratchDir::new(label);
12932            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12933            let supervisor = Supervisor::new_for_test(
12934                Arc::new(Registry::default()),
12935                RestartPolicy::new(3, Duration::ZERO),
12936            );
12937            let module = supervisor.spawn(spec).unwrap();
12938            wait_for_file(&ready).await;
12939
12940            if disable {
12941                module.set_enabled(false).await.unwrap();
12942            } else {
12943                module.stop().await.unwrap();
12944            }
12945            assert!(
12946                marker.exists(),
12947                "{label}: the child must have left through its SIGTERM handler with exit 0"
12948            );
12949
12950            // Long enough for a zero-backoff respawn to have happened if the
12951            // exit had been treated as a crash.
12952            sleep(Duration::from_millis(500)).await;
12953            let status = module.status().unwrap();
12954            let expected = if disable {
12955                ModuleState::Disabled
12956            } else {
12957                ModuleState::Stopped
12958            };
12959            assert_eq!(status.state, expected, "{label}");
12960            assert_eq!(
12961                status.spawn_generation, 1,
12962                "{label}: respawned after a requested stop"
12963            );
12964            let history = module.terminal_history();
12965            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12966            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12967            assert_ne!(
12968                history.entries[0].disposition,
12969                TerminalDisposition::Restarting,
12970                "{label}"
12971            );
12972        }
12973    }
12974
12975    /// A subc-wire module that exits 0 on its own is still a stop: the
12976    /// protocol-none rule must not reach it.
12977    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12978    async fn subc_wire_clean_exit_is_still_a_stop() {
12979        let supervisor = Supervisor::new_for_test(
12980            Arc::new(Registry::default()),
12981            RestartPolicy::new(3, Duration::ZERO),
12982        );
12983        let module = supervisor
12984            .spawn(ModuleSpec {
12985                module_id: "wire-clean-exit".to_string(),
12986                program: fake_aft_stub_path(),
12987                args: Vec::new(),
12988                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12989                reserved: false,
12990                reserved_prefixes: Vec::new(),
12991                protocol: ModuleProtocol::Subc,
12992                overlap: Default::default(),
12993            })
12994            .unwrap();
12995
12996        let deadline = Instant::now() + Duration::from_secs(10);
12997        while module.terminal_history().entries.is_empty() {
12998            assert!(Instant::now() < deadline, "module never exited");
12999            sleep(Duration::from_millis(10)).await;
13000        }
13001        // Long enough for a zero-backoff respawn to have happened.
13002        sleep(Duration::from_millis(500)).await;
13003        let status = module.status().unwrap();
13004        assert_eq!(status.state, ModuleState::Stopped);
13005        assert_eq!(status.spawn_generation, 1);
13006        let history = module.terminal_history();
13007        assert_eq!(history.entries.len(), 1, "{history:?}");
13008        assert_eq!(history.entries[0].exit_code, Some(0));
13009        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13010    }
13011
13012    #[cfg(unix)]
13013    #[tokio::test]
13014    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13015        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13016        let record = dir.join("live-children.json");
13017        let supervisor = Supervisor::new_for_test(
13018            Arc::new(Registry::default()),
13019            RestartPolicy::new(0, Duration::ZERO),
13020        );
13021        let mut runtime = supervisor.runtime_config();
13022        runtime.child_roster.record_to(record.clone());
13023        let gate = Arc::new(super::ReloadExitRecordGate::default());
13024        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13025        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13026        let spec = ModuleSpec {
13027            module_id: "reload-exit-roster".into(),
13028            program: fake_aft_stub_path(),
13029            args: Vec::new(),
13030            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13031            reserved: false,
13032            reserved_prefixes: Vec::new(),
13033            protocol: ModuleProtocol::Subc,
13034            overlap: Default::default(),
13035        };
13036        let mut child = None;
13037        let reload = super::finish_reload_child(
13038            &spec,
13039            &runtime,
13040            &supervisor.registry,
13041            &supervisor.process_liveness,
13042            &snapshot,
13043            &mut child,
13044        );
13045        tokio::pin!(reload);
13046        tokio::select! {
13047            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13048            _ = gate.reached.notified() => {}
13049        }
13050        assert!(runtime
13051            .terminal_ring
13052            .lock()
13053            .unwrap()
13054            .snapshot()
13055            .entries
13056            .is_empty());
13057        assert_eq!(
13058            crate::live_children::read_record(&record).unwrap().len(),
13059            1,
13060            "shutdown must still wait for the reaped child until its terminal record exists"
13061        );
13062        runtime.child_roster.close();
13063        gate.resume.notify_one();
13064        assert!(reload.await.is_err());
13065        assert!(crate::live_children::read_record(&record)
13066            .unwrap()
13067            .is_empty());
13068        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13069        assert_eq!(history.entries.len(), 1);
13070        assert_eq!(
13071            history.entries[0].disposition,
13072            TerminalDisposition::DaemonShutdown
13073        );
13074    }
13075
13076    /// Each restart-producing arm has its own state transition. Keeping their
13077    /// lifetime count assertions adjacent prevents a later new arm from silently
13078    /// spending budget without recording the historical restart.
13079    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13080    async fn every_restart_increment_path_advances_lifetime_count() {
13081        let supervisor = Supervisor::new_for_test(
13082            Arc::new(Registry::default()),
13083            RestartPolicy::new(1, Duration::ZERO),
13084        );
13085        let runtime = supervisor.runtime_config();
13086        let spec = ModuleSpec {
13087            module_id: "lifetime-increment-path".to_string(),
13088            program: PathBuf::from("/unused/lifetime-increment-path"),
13089            args: Vec::new(),
13090            env: Vec::new(),
13091            reserved: false,
13092            reserved_prefixes: Vec::new(),
13093            protocol: ModuleProtocol::Subc,
13094            overlap: Default::default(),
13095        };
13096
13097        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13098        assert!(matches!(
13099            on_child_exit(
13100                &spec,
13101                runtime.restart_policy,
13102                &supervisor.registry,
13103                &crash_snapshot,
13104                &runtime.terminal_ring,
13105                &runtime.spawn_events,
13106                &runtime.child_roster,
13107                ExitReport {
13108                    kind: ExitKind::Crash,
13109                    code: Some(1),
13110                    signal: None,
13111                    at_ms: 1,
13112                },
13113                true,
13114            )
13115            .await,
13116            NextAction::Restart { schedule: _ }
13117        ));
13118        let (crash_restarts, crash_lifetime) = {
13119            let state = lock_snapshot(&crash_snapshot).unwrap();
13120            (state.crash_restarts.len(), state.lifetime_restarts)
13121        };
13122        assert_eq!(crash_restarts, 1);
13123        assert_eq!(crash_lifetime, 1);
13124
13125        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13126        let mut health_child = None;
13127        assert!(matches!(
13128            health_restart_child(
13129                &spec,
13130                &runtime,
13131                &supervisor.registry,
13132                &supervisor.process_liveness,
13133                &health_snapshot,
13134                &mut health_child,
13135                SupervisorHealthStatus::Failing,
13136                None,
13137                2,
13138            )
13139            .await,
13140            Ok(())
13141        ));
13142        assert!(health_child.is_none());
13143        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13144        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13145        let (health_restarts, health_lifetime) = {
13146            let state = lock_snapshot(&health_snapshot).unwrap();
13147            (state.crash_restarts.len(), state.lifetime_restarts)
13148        };
13149        assert_eq!(health_restarts, 1);
13150        assert_eq!(health_lifetime, 1);
13151
13152        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13153        let mut reload_child = None;
13154        assert!(matches!(
13155            handle_reload_spawn_failure(
13156                &spec,
13157                &runtime,
13158                &supervisor.process_liveness,
13159                &reload_snapshot,
13160                &mut reload_child,
13161                "forced reload spawn failure".to_string(),
13162            )
13163            .await,
13164            Err(SuperviseError::ReloadFailed { .. })
13165        ));
13166        let (reload_restarts, reload_lifetime) = {
13167            let state = lock_snapshot(&reload_snapshot).unwrap();
13168            (state.crash_restarts.len(), state.lifetime_restarts)
13169        };
13170        assert_eq!(reload_restarts, 1);
13171        assert_eq!(reload_lifetime, 1);
13172    }
13173
13174    #[tokio::test]
13175    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13176        let supervisor = Supervisor::new_for_test(
13177            Arc::new(Registry::default()),
13178            RestartPolicy::new(3, Duration::ZERO),
13179        );
13180        let runtime = supervisor.runtime_config();
13181        let spec = ModuleSpec {
13182            module_id: "deliberately-severed".to_string(),
13183            program: PathBuf::from("/unused/deliberately-severed"),
13184            args: Vec::new(),
13185            env: Vec::new(),
13186            reserved: false,
13187            reserved_prefixes: Vec::new(),
13188            protocol: ModuleProtocol::Subc,
13189            overlap: Default::default(),
13190        };
13191        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13192        let process = ProcessIdentity {
13193            pid: 41,
13194            start_time: 101,
13195        };
13196        record_deliberate_severance(&snapshot, process).unwrap();
13197        let exit_report = apply_deliberate_severance_marker(
13198            &snapshot,
13199            Some(process),
13200            ExitReport {
13201                kind: ExitKind::Crash,
13202                code: Some(1),
13203                signal: None,
13204                at_ms: 1,
13205            },
13206        );
13207        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13208
13209        assert!(matches!(
13210            on_child_exit(
13211                &spec,
13212                runtime.restart_policy,
13213                &supervisor.registry,
13214                &snapshot,
13215                &runtime.terminal_ring,
13216                &runtime.spawn_events,
13217                &runtime.child_roster,
13218                exit_report,
13219                true,
13220            )
13221            .await,
13222            NextAction::Restart { schedule: _ }
13223        ));
13224        let state = lock_snapshot(&snapshot).unwrap();
13225        assert_eq!(state.lifetime_restarts, 1);
13226        assert_eq!(state.crash_restarts.len(), 0);
13227    }
13228
13229    #[tokio::test]
13230    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13231        let supervisor = Supervisor::new_for_test(
13232            Arc::new(Registry::default()),
13233            RestartPolicy::new(3, Duration::ZERO),
13234        );
13235        let runtime = supervisor.runtime_config();
13236        let spec = ModuleSpec {
13237            module_id: "genuine-crash".to_string(),
13238            program: PathBuf::from("/unused/genuine-crash"),
13239            args: Vec::new(),
13240            env: Vec::new(),
13241            reserved: false,
13242            reserved_prefixes: Vec::new(),
13243            protocol: ModuleProtocol::Subc,
13244            overlap: Default::default(),
13245        };
13246        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13247
13248        assert!(matches!(
13249            on_child_exit(
13250                &spec,
13251                runtime.restart_policy,
13252                &supervisor.registry,
13253                &snapshot,
13254                &runtime.terminal_ring,
13255                &runtime.spawn_events,
13256                &runtime.child_roster,
13257                ExitReport {
13258                    kind: ExitKind::Crash,
13259                    code: Some(1),
13260                    signal: None,
13261                    at_ms: 1,
13262                },
13263                true,
13264            )
13265            .await,
13266            NextAction::Restart { schedule: _ }
13267        ));
13268        let state = lock_snapshot(&snapshot).unwrap();
13269        assert_eq!(state.lifetime_restarts, 1);
13270        assert_eq!(state.crash_restarts.len(), 1);
13271    }
13272
13273    fn crash_exit_report(at_ms: u64) -> ExitReport {
13274        ExitReport {
13275            kind: ExitKind::Crash,
13276            code: Some(1),
13277            signal: None,
13278            at_ms,
13279        }
13280    }
13281
13282    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13283        ModuleSpec {
13284            module_id: module_id.to_string(),
13285            program: PathBuf::from("/unused").join(module_id),
13286            args: Vec::new(),
13287            env: Vec::new(),
13288            reserved: false,
13289            reserved_prefixes: Vec::new(),
13290            protocol: ModuleProtocol::Subc,
13291            overlap: Default::default(),
13292        }
13293    }
13294
13295    /// A real crash loop still stops. Three crashes with nothing aging out spend
13296    /// a budget of two and the third respawn is refused, and both surfaces an
13297    /// operator has -- the log line and the retained terminal record -- name the
13298    /// window rather than only the cap, because `max_restarts=2` alone is what
13299    /// this budget used to mean.
13300    #[tokio::test]
13301    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13302        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13303        let supervisor = Supervisor::new_for_test(
13304            Arc::new(Registry::default()),
13305            RestartPolicy::new(2, Duration::ZERO),
13306        );
13307        let runtime = supervisor.runtime_config();
13308        let spec = windowed_crash_spec("crash-loop-in-window");
13309        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13310
13311        for attempt in 1..=2 {
13312            assert!(
13313                matches!(
13314                    on_child_exit(
13315                        &spec,
13316                        runtime.restart_policy,
13317                        &supervisor.registry,
13318                        &snapshot,
13319                        &runtime.terminal_ring,
13320                        &runtime.spawn_events,
13321                        &runtime.child_roster,
13322                        crash_exit_report(attempt),
13323                        true,
13324                    )
13325                    .await,
13326                    NextAction::Restart { schedule: _ }
13327                ),
13328                "crash {attempt} is inside the budget and must respawn"
13329            );
13330        }
13331
13332        assert!(matches!(
13333            on_child_exit(
13334                &spec,
13335                runtime.restart_policy,
13336                &supervisor.registry,
13337                &snapshot,
13338                &runtime.terminal_ring,
13339                &runtime.spawn_events,
13340                &runtime.child_roster,
13341                crash_exit_report(3),
13342                true,
13343            )
13344            .await,
13345            NextAction::Stop { .. }
13346        ));
13347
13348        {
13349            let state = lock_snapshot(&snapshot).unwrap();
13350            assert_eq!(state.state, ModuleState::Failed);
13351            assert_eq!(state.crash_restarts.len(), 2);
13352            assert_eq!(state.lifetime_restarts, 2);
13353        }
13354
13355        let history = runtime
13356            .terminal_ring
13357            .lock()
13358            .expect("terminal ring is not poisoned")
13359            .snapshot();
13360        let last = history
13361            .entries
13362            .last()
13363            .expect("the refused crash is retained");
13364        assert_eq!(last.disposition, TerminalDisposition::Failed);
13365        assert_eq!(
13366            last.disposition_detail.as_deref(),
13367            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13368        );
13369
13370        let captured = crate::router::test_log::captured_logs(&logs);
13371        assert!(
13372            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13373            "the stop must be logged with its window: {captured}"
13374        );
13375    }
13376
13377    /// The rate, stated as a test: three crashes where the first has aged past
13378    /// the window are two crashes as far as the budget is concerned, so the
13379    /// third respawn is allowed and the ring holds only the two recent ones.
13380    ///
13381    /// This is the case a lifetime counter got wrong -- and the case the daemon
13382    /// now hits routinely, since a module exits non-zero every time its
13383    /// connection to the daemon drops.
13384    #[tokio::test]
13385    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13386        let supervisor = Supervisor::new_for_test(
13387            Arc::new(Registry::default()),
13388            RestartPolicy::new(2, Duration::ZERO),
13389        );
13390        let runtime = supervisor.runtime_config();
13391        let spec = windowed_crash_spec("crash-across-windows");
13392        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13393
13394        for attempt in 1..=2 {
13395            assert!(matches!(
13396                on_child_exit(
13397                    &spec,
13398                    runtime.restart_policy,
13399                    &supervisor.registry,
13400                    &snapshot,
13401                    &runtime.terminal_ring,
13402                    &runtime.spawn_events,
13403                    &runtime.child_roster,
13404                    crash_exit_report(attempt),
13405                    true,
13406                )
13407                .await,
13408                NextAction::Restart { schedule: _ }
13409            ));
13410        }
13411
13412        // The oldest crash moves out of the window; nothing else about the
13413        // module changes.
13414        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13415            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13416        })
13417        .unwrap();
13418
13419        assert!(
13420            matches!(
13421                on_child_exit(
13422                    &spec,
13423                    runtime.restart_policy,
13424                    &supervisor.registry,
13425                    &snapshot,
13426                    &runtime.terminal_ring,
13427                    &runtime.spawn_events,
13428                    &runtime.child_roster,
13429                    crash_exit_report(3),
13430                    true,
13431                )
13432                .await,
13433                NextAction::Restart { schedule: _ }
13434            ),
13435            "a crash older than the window must not hold a budget slot"
13436        );
13437
13438        let state = lock_snapshot(&snapshot).unwrap();
13439        assert_eq!(state.state, ModuleState::Restarting);
13440        assert_eq!(
13441            state.crash_restarts.len(),
13442            2,
13443            "the aged instant is dropped and the new one takes its place"
13444        );
13445        assert_eq!(
13446            state.lifetime_restarts, 3,
13447            "the ledger counts every restart, including the ones the window forgot"
13448        );
13449    }
13450
13451    /// An operator restart hands the budget back whole, and the ledger keeps
13452    /// counting. Those are different questions -- "how close is this module to
13453    /// being stopped" and "how many times has it been replaced" -- and the
13454    /// operator action answers only the first.
13455    #[tokio::test]
13456    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13457        let supervisor = Supervisor::new_for_test(
13458            Arc::new(Registry::default()),
13459            RestartPolicy::new(2, Duration::ZERO),
13460        );
13461        let runtime = supervisor.runtime_config();
13462        let spec = windowed_crash_spec("operator-cleared-budget");
13463        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13464
13465        for attempt in 1..=2 {
13466            assert!(matches!(
13467                on_child_exit(
13468                    &spec,
13469                    runtime.restart_policy,
13470                    &supervisor.registry,
13471                    &snapshot,
13472                    &runtime.terminal_ring,
13473                    &runtime.spawn_events,
13474                    &runtime.child_roster,
13475                    crash_exit_report(attempt),
13476                    true,
13477                )
13478                .await,
13479                NextAction::Restart { schedule: _ }
13480            ));
13481        }
13482
13483        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13484        {
13485            let state = lock_snapshot(&snapshot).unwrap();
13486            assert!(
13487                state.crash_restarts.is_empty(),
13488                "an operator restart returns the full budget"
13489            );
13490            assert_eq!(
13491                state.lifetime_restarts, 2,
13492                "clearing the budget must not unmake the crashes"
13493            );
13494        }
13495
13496        assert!(
13497            matches!(
13498                on_child_exit(
13499                    &spec,
13500                    runtime.restart_policy,
13501                    &supervisor.registry,
13502                    &snapshot,
13503                    &runtime.terminal_ring,
13504                    &runtime.spawn_events,
13505                    &runtime.child_roster,
13506                    crash_exit_report(3),
13507                    true,
13508                )
13509                .await,
13510                NextAction::Restart { schedule: _ }
13511            ),
13512            "the cleared budget must be spendable again"
13513        );
13514        let state = lock_snapshot(&snapshot).unwrap();
13515        assert_eq!(state.crash_restarts.len(), 1);
13516        assert_eq!(state.lifetime_restarts, 3);
13517    }
13518
13519    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13520    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13521        let severed = ProcessIdentity {
13522            pid: 41,
13523            start_time: 101,
13524        };
13525        let successor = ProcessIdentity {
13526            pid: 41,
13527            start_time: 202,
13528        };
13529        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13530        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13531            state.pid = Some(successor.pid);
13532            state.process_start_time = Some(successor.start_time);
13533        })
13534        .unwrap();
13535        assert!(!module.record_deliberate_severance(severed).unwrap());
13536
13537        let exit_report = apply_deliberate_severance_marker(
13538            &module.inner.snapshot,
13539            Some(successor),
13540            ExitReport {
13541                kind: ExitKind::Crash,
13542                code: Some(1),
13543                signal: None,
13544                at_ms: 1,
13545            },
13546        );
13547
13548        assert_eq!(exit_report.kind, ExitKind::Crash);
13549    }
13550
13551    #[tokio::test]
13552    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13553        let registry = Registry::default();
13554        let supervisor = Supervisor::new_for_test(
13555            Arc::new(Registry::default()),
13556            RestartPolicy::new(3, Duration::ZERO),
13557        );
13558        let runtime = supervisor.runtime_config();
13559        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13560        let spec = ModuleSpec {
13561            module_id: "drain-deliberate-severance".to_string(),
13562            program: fake_aft_stub_path(),
13563            args: Vec::new(),
13564            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13565            reserved: false,
13566            reserved_prefixes: Vec::new(),
13567            protocol: ModuleProtocol::Subc,
13568            overlap: Default::default(),
13569        };
13570        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13571        let process = ProcessIdentity {
13572            pid: 41,
13573            start_time: 101,
13574        };
13575        child.process_identity = Some(process);
13576        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13577            state.pid = Some(process.pid);
13578            state.process_start_time = Some(process.start_time);
13579        })
13580        .unwrap();
13581        record_deliberate_severance(&snapshot, process).unwrap();
13582
13583        drain_child_to_state(
13584            &spec.module_id,
13585            spec.protocol,
13586            // The child exits on its own; no signal may change the exit this
13587            // test classifies.
13588            StopNotice::SentOverConnection,
13589            &registry,
13590            None,
13591            &snapshot,
13592            &runtime.terminal_ring,
13593            &runtime.spawn_events,
13594            child,
13595            Duration::from_secs(1),
13596            ModuleState::Stopped,
13597            Some(false),
13598            &super::FailureContext::operator(),
13599        )
13600        .await
13601        .unwrap();
13602
13603        let state = lock_snapshot(&snapshot).unwrap();
13604        assert_eq!(
13605            state.last_exit.as_ref().map(|exit| exit.kind),
13606            Some(ExitKind::DeliberateSeverance)
13607        );
13608        assert_eq!(state.lifetime_restarts, 1);
13609        assert_eq!(state.crash_restarts.len(), 0);
13610        drop(state);
13611        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13612        assert_eq!(
13613            history.entries[0].exit_kind,
13614            subc_control::TerminalExitKind::DeliberateSeverance
13615        );
13616    }
13617
13618    #[tokio::test]
13619    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13620        let registry = Registry::default();
13621        let supervisor = Supervisor::new_for_test(
13622            Arc::new(Registry::default()),
13623            RestartPolicy::new(3, Duration::ZERO),
13624        );
13625        let runtime = supervisor.runtime_config();
13626        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13627        let spec = ModuleSpec {
13628            module_id: "ordinary-drain".to_string(),
13629            program: fake_aft_stub_path(),
13630            args: Vec::new(),
13631            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13632            reserved: false,
13633            reserved_prefixes: Vec::new(),
13634            protocol: ModuleProtocol::Subc,
13635            overlap: Default::default(),
13636        };
13637        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13638
13639        drain_child_to_state(
13640            &spec.module_id,
13641            spec.protocol,
13642            // The child exits on its own; no signal may change the exit this
13643            // test classifies.
13644            StopNotice::SentOverConnection,
13645            &registry,
13646            None,
13647            &snapshot,
13648            &runtime.terminal_ring,
13649            &runtime.spawn_events,
13650            child,
13651            Duration::from_secs(1),
13652            ModuleState::Stopped,
13653            Some(false),
13654            &super::FailureContext::operator(),
13655        )
13656        .await
13657        .unwrap();
13658
13659        let state = lock_snapshot(&snapshot).unwrap();
13660        assert_eq!(
13661            state.last_exit.as_ref().map(|exit| exit.kind),
13662            Some(ExitKind::Crash)
13663        );
13664        assert_eq!(state.lifetime_restarts, 0);
13665        assert_eq!(state.crash_restarts.len(), 0);
13666    }
13667
13668    #[test]
13669    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13670        // The server's generic fatal-routing branch only knows that the
13671        // connection failed; it does not know that the daemon deliberately
13672        // initiated a process-killing severance. Keep this seam explicit so a
13673        // future connection error path cannot silently reintroduce the stale
13674        // exemption that mislabels a later genuine crash.
13675        assert!(!include_str!("server.rs")
13676            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13677    }
13678
13679    /// The `route.closed` `drained` value must be the quiescence wait's own
13680    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13681    /// measurement at all and `false` is the one honest constant. This is the exact
13682    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13683    /// on every return path, including the one that used to return early via `?`
13684    /// with `route.closing` already sent and no `route.closed` ever following.
13685    #[test]
13686    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13687        assert!(drained_after_quiescence_wait(&Ok(true)));
13688        assert!(!drained_after_quiescence_wait(&Ok(false)));
13689        assert!(!drained_after_quiescence_wait(&Err(
13690            SuperviseError::StatePoisoned { module_id: None }
13691        )));
13692    }
13693
13694    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13695    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13696    /// already reaped out-of-band) still leaves a terminal record rather than none
13697    /// at all. Triggering the real `wait()` I/O error from an integration test would
13698    /// need a genuine already-reaped-child race, which is OS-specific and not
13699    /// something this suite attempts elsewhere; this test instead verifies the
13700    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13701    /// the call site itself is verified by inspection to sit in that exact arm.
13702    #[test]
13703    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13704        let ring = Arc::new(Mutex::new(TerminalRing::new(
13705            TerminalRingConfig::default(),
13706            0,
13707        )));
13708        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13709
13710        let snapshot = ring.lock().unwrap().snapshot();
13711        assert_eq!(snapshot.entries.len(), 1);
13712        let entry = &snapshot.entries[0];
13713        assert_eq!(entry.exit_code, None);
13714        assert_eq!(entry.exit_signal, None);
13715        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13716    }
13717
13718    #[test]
13719    fn wait_error_exit_path_preserves_spawn_event_density() {
13720        let feed = super::SpawnEventFeed::default();
13721        feed.configure_incarnation("wait-error-density".to_string());
13722        feed.emit_spawned("wait-error", 41, 1);
13723        let ring = Arc::new(Mutex::new(TerminalRing::new(
13724            TerminalRingConfig::default(),
13725            0,
13726        )));
13727
13728        record_wait_error_terminal("wait-error", &ring, &feed);
13729        feed.emit_spawned("after-wait-error", 42, 2);
13730
13731        let state = feed.0.lock().unwrap();
13732        let sequences = state
13733            .events
13734            .iter()
13735            .map(|event| event.cursor.seq)
13736            .collect::<Vec<_>>();
13737        assert_eq!(sequences, vec![1, 2, 3]);
13738        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13739        assert_eq!(state.events[1].exit_code, None);
13740        assert_eq!(state.events[1].exit_signal, None);
13741    }
13742
13743    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13744    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13745    /// not a clean exit it never actually observed.
13746    #[test]
13747    fn wait_error_exit_report_is_classified_as_a_crash() {
13748        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13749    }
13750}
13751
13752#[cfg(all(test, unix))]
13753mod restart_ordering_tests {
13754    use super::*;
13755    use std::io::Write;
13756    use tokio::io::AsyncReadExt;
13757
13758    #[test]
13759    #[ignore = "child process fixture"]
13760    fn serving_child() {
13761        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13762            return;
13763        };
13764        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13765        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13766        for stream in listener.incoming() {
13767            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13768        }
13769    }
13770
13771    async fn restart_before_publication(requests: usize) {
13772        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13773        let address_file = home.path().join("address");
13774        let spec = ModuleSpec {
13775            module_id: "restart-serving-child".into(),
13776            program: std::env::current_exe().unwrap(),
13777            args: vec![
13778                "--exact".into(),
13779                "supervise::restart_ordering_tests::serving_child".into(),
13780                "--ignored".into(),
13781                "--nocapture".into(),
13782            ],
13783            env: vec![(
13784                "RESTART_TEST_ADDRESS_FILE".into(),
13785                address_file.display().to_string(),
13786            )],
13787            reserved: false,
13788            reserved_prefixes: vec![],
13789            protocol: ModuleProtocol::None,
13790            overlap: Default::default(),
13791        };
13792        let mut supervisor =
13793            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13794        let (spawned_tx, spawned_rx) = oneshot::channel();
13795        let spawned_tx = Mutex::new(Some(spawned_tx));
13796        let (release_tx, release_rx) = std::sync::mpsc::channel();
13797        let release_rx = Mutex::new(release_rx);
13798        // Hold publication, not the child: it must answer a real request before
13799        // the restart enters the queue, even though status still has generation 0.
13800        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13801            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13802            release_rx
13803                .lock()
13804                .unwrap()
13805                .recv_timeout(Duration::from_secs(20))
13806                .unwrap();
13807        })));
13808        let registered = supervisor
13809            .register_configured_with_health(
13810                spec.clone(),
13811                true,
13812                HealthConfig::default(),
13813                None,
13814                RestartPolicy::new(3, Duration::ZERO),
13815            )
13816            .unwrap();
13817        let module = registered.module.clone();
13818        let startup = tokio::task::spawn_blocking(move || {
13819            supervisor.start_registered_configured(registered, None, true)
13820        });
13821        spawned_rx.await.unwrap();
13822        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13823            let address = loop {
13824                if let Ok(address) = std::fs::read_to_string(&address_file) {
13825                    if !address.is_empty() {
13826                        break address;
13827                    }
13828                }
13829                tokio::time::sleep(Duration::from_millis(1)).await;
13830            };
13831            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13832            let mut reply = String::new();
13833            stream.read_to_string(&mut reply).await.unwrap();
13834            reply.trim().parse::<u32>().unwrap()
13835        })
13836        .await
13837        .expect("child must serve before generation publication");
13838        assert_eq!(
13839            lock_snapshot(&module.inner.snapshot)
13840                .unwrap()
13841                .spawn_generation,
13842            0
13843        );
13844        let mut replies = Vec::new();
13845        for _ in 0..requests {
13846            let mut restart = Box::pin(module.restart(Some(0)));
13847            // Poll the public API through queue admission, leaving only its ACK
13848            // pending. The monitor cannot consume the command until released.
13849            std::future::poll_fn(|cx| {
13850                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13851                std::task::Poll::Ready(())
13852            })
13853            .await;
13854            replies.push(restart);
13855        }
13856        release_tx.send(()).unwrap();
13857        startup.await.unwrap().unwrap();
13858        for reply in replies {
13859            reply.await.unwrap();
13860        }
13861        // Restart schedules its replacement after completing the handler. Wait
13862        // for that separate actor turn before placing the command barrier.
13863        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13864            loop {
13865                if lock_snapshot(&module.inner.snapshot)
13866                    .unwrap()
13867                    .spawn_generation
13868                    > 1
13869                {
13870                    break;
13871                }
13872                tokio::time::sleep(Duration::from_millis(1)).await;
13873            }
13874        })
13875        .await;
13876        // Commands are handled serially. The update-spec reply proves every
13877        // restart handler has finished before we inspect the snapshot; restart's
13878        // own ACK only acknowledges initiation, not replacement completion.
13879        module.update_spec_for_test(spec).await.unwrap();
13880        let (generation, pid) = {
13881            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13882            (state.spawn_generation, state.pid)
13883        };
13884        module.stop().await.unwrap();
13885        replacement.expect("restart must publish a replacement for the already-serving child");
13886        assert_ne!(
13887            pid,
13888            Some(old_pid),
13889            "a serving child predating the restart must be replaced"
13890        );
13891        assert_eq!(generation, 2, "queued restarts must share one replacement");
13892    }
13893
13894    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13895    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13896        restart_before_publication(1).await;
13897    }
13898
13899    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13900    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13901        restart_before_publication(2).await;
13902    }
13903
13904    #[tokio::test]
13905    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13906        let supervisor =
13907            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13908        let registered = supervisor
13909            .register_configured_with_health(
13910                ModuleSpec {
13911                    module_id: "queued-before-replacement".into(),
13912                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
13913                    // macOS a module whose running image is not the configured
13914                    // program is refused once its exec is confirmed (see
13915                    // `record_write_stall_tests::sleeping_spec`).
13916                    program: "/bin/sleep".into(),
13917                    args: vec!["600".into()],
13918                    env: vec![],
13919                    reserved: false,
13920                    reserved_prefixes: vec![],
13921                    protocol: ModuleProtocol::None,
13922                    overlap: Default::default(),
13923                },
13924                true,
13925                HealthConfig::default(),
13926                None,
13927                RestartPolicy::default(),
13928            )
13929            .unwrap();
13930        let RegisteredConfiguredModule {
13931            mut spec,
13932            mut runtime,
13933            module,
13934            mut commands,
13935        } = registered;
13936        let snapshot = &module.inner.snapshot;
13937        let mut restart = Box::pin(module.restart(Some(0)));
13938        std::future::poll_fn(|cx| {
13939            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13940            std::task::Poll::Ready(())
13941        })
13942        .await;
13943        // Drive the actor explicitly so a replacement is definitely spawned
13944        // after receipt, but before this queued command is dequeued.
13945        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13946        let replacement_pid = child.as_ref().unwrap().id();
13947        let command = commands.recv().await.unwrap();
13948        assert!(
13949            handle_supervisor_command(
13950                command,
13951                &mut spec,
13952                &mut runtime,
13953                &supervisor.registry,
13954                &supervisor.process_liveness,
13955                snapshot,
13956                &mut child,
13957                &mut commands,
13958                &mut VecDeque::new(),
13959            )
13960            .await
13961        );
13962        restart.await.unwrap();
13963        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13964        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13965        if let Some(mut child) = child {
13966            child.start_kill().unwrap();
13967            child.wait().await.unwrap();
13968        }
13969        assert_eq!(
13970            kept_pid, replacement_pid,
13971            "a newer replacement must survive"
13972        );
13973        assert!(
13974            !pending,
13975            "a satisfied restart must not schedule another spawn"
13976        );
13977    }
13978}
13979
13980#[cfg(all(test, unix))]
13981mod health_event_tests {
13982    use super::*;
13983    use cortexkit_test_support::ScratchDir;
13984    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13985
13986    struct Actor {
13987        module: SupervisedModule,
13988        registry: Arc<Registry>,
13989        forwarding: Arc<ForwardingTable>,
13990        spec: ModuleSpec,
13991        health: HealthConfig,
13992        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13993        _home: ScratchDir,
13994    }
13995
13996    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13997        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13998        // for an observed condition, rather than letting paused time auto-advance
13999        // deadlines or assuming a fixed number of yields finishes the work.
14000        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14001        let tick = Instant::now();
14002        loop {
14003            if let Some(value) = observe() {
14004                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14005                return value;
14006            }
14007            assert!(
14008                std::time::Instant::now() < deadline,
14009                "timed out waiting for {reason}"
14010            );
14011            tokio::task::yield_now().await;
14012        }
14013    }
14014
14015    impl Actor {
14016        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14017            let home = ScratchDir::new("health-events");
14018            let registry = Arc::new(Registry::default());
14019            let forwarding = Arc::new(ForwardingTable::default());
14020            let health = HealthConfig {
14021                cadence,
14022                deadline: Duration::from_secs(3600),
14023                ..HealthConfig::default()
14024            };
14025            let supervisor =
14026                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
14027                    .with_forwarding(forwarding.clone())
14028                    .with_health_config(health.clone());
14029            let spec = ModuleSpec {
14030                module_id: "event-health-child".into(),
14031                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14032                // the privacy trampoline refuses a module whose running image
14033                // is not the configured program, and `start` below waits for
14034                // the process to be confirmed (`reported_pid`). The shell form
14035                // passed only while the image check ran before the shell had
14036                // replaced itself with `sleep` (see
14037                // `record_write_stall_tests::sleeping_spec`).
14038                program: PathBuf::from("/bin/sleep"),
14039                // The process stays alive but has no real bus peer. Registrations
14040                // below exercise the same registry writes as accepted HELLOs.
14041                args: vec!["600".into()],
14042                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14043                    .into_iter()
14044                    .map(|key| (key.into(), home.path().display().to_string()))
14045                    .collect(),
14046                reserved: false,
14047                reserved_prefixes: vec![],
14048                protocol,
14049                overlap: Default::default(),
14050            };
14051            let module = supervisor.spawn(spec.clone()).unwrap();
14052            let (_, rx) = mpsc::channel(8);
14053            let actor = Self {
14054                module,
14055                registry,
14056                forwarding,
14057                spec,
14058                health,
14059                rx,
14060                _home: home,
14061            };
14062            actor
14063                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14064                .await;
14065            actor
14066        }
14067
14068        fn turns(&self) -> u64 {
14069            lock_snapshot(&self.module.inner.snapshot)
14070                .unwrap()
14071                .actor_turns
14072        }
14073
14074        async fn wait_for_select(
14075            &self,
14076            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14077        ) -> ActorSelectCheckpoint {
14078            wait_for("actor select checkpoint", || {
14079                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14080                state.actor_select.filter(|checkpoint| {
14081                    checkpoint.generation == state.spawn_generation
14082                        && state.state == ModuleState::Running
14083                        && state.process_alive
14084                        && state.reported_pid().is_some()
14085                        && expected(checkpoint)
14086                })
14087            })
14088            .await
14089        }
14090
14091        fn hello(&mut self, connection: u64, health: bool) {
14092            let manifest =
14093                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14094                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14095                    .build();
14096            let connection = ConnectionId::new(connection);
14097            let (tx, rx) = mpsc::channel(8);
14098            self.rx = rx;
14099            self.forwarding
14100                .register_module_connection(
14101                    connection,
14102                    self.spec.module_id.clone(),
14103                    subc_protocol::PROTOCOL_VERSION,
14104                    Concurrency::ModuleManaged,
14105                    FrameSink::new(tx),
14106                )
14107                .unwrap();
14108            self.registry
14109                .register_with_control_ops(
14110                    manifest,
14111                    subc_protocol::PROTOCOL_VERSION,
14112                    connection,
14113                    if health {
14114                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14115                    } else {
14116                        vec![]
14117                    },
14118                )
14119                .unwrap();
14120        }
14121
14122        async fn probe(&mut self) -> crate::router::OutboundFrame {
14123            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14124                Ok(frame) => Some(frame),
14125                Err(mpsc::error::TryRecvError::Empty) => None,
14126                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14127            })
14128            .await;
14129            let body: Value = serde_json::from_slice(&frame.body).unwrap();
14130            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14131            frame
14132        }
14133
14134        fn no_probe(&mut self) {
14135            assert!(matches!(
14136                self.rx.try_recv(),
14137                Err(mpsc::error::TryRecvError::Empty)
14138            ));
14139        }
14140
14141        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14142            self.forwarding
14143                .complete_module_control_rpc(
14144                    ConnectionId::new(connection),
14145                    frame.header.corr,
14146                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14147                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14148                        status: HealthStatus::Ok,
14149                        detail: None,
14150                        metrics: None,
14151                    }),
14152                )
14153                .unwrap();
14154        }
14155    }
14156
14157    #[tokio::test(start_paused = true)]
14158    async fn no_health_children_park_for_sixty_seconds() {
14159        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14160        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14161        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14162        unadvertised.hello(1, false);
14163        // Include any one-off registration wake in the budget; receiving a
14164        // no-health HELLO must not turn a parked child into a polling child.
14165        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
14166        // a single 60 s jump would only observe one expired timer.
14167        for _ in 0..6000 {
14168            tokio::time::advance(Duration::from_millis(10)).await;
14169            tokio::task::yield_now().await;
14170        }
14171        for actor in [&none, &unregistered, &unadvertised] {
14172            assert!(
14173                actor.turns() <= 5,
14174                "no-health actor woke {} times",
14175                actor.turns()
14176            );
14177            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14178        }
14179    }
14180
14181    #[tokio::test(start_paused = true)]
14182    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14183        // With a zero cadence the first probe is due at once, so this measures
14184        // only how fast the registration notification starts health probing,
14185        // not the normal 30-33 s wait before a first probe (checked below).
14186        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14187        let tick = Instant::now();
14188        actor.hello(1, true);
14189        actor.probe().await;
14190        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14191    }
14192
14193    #[tokio::test(start_paused = true)]
14194    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14195        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14196        let hello_tick = Instant::now();
14197        actor.hello(1, true);
14198        let armed = actor
14199            .wait_for_select(|checkpoint| {
14200                checkpoint.registered_connection == Some(ConnectionId::new(1))
14201                    && checkpoint.next_probe_at.is_some()
14202            })
14203            .await;
14204        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14205        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14206        let first_deadline = hello_tick + delay;
14207        assert_eq!(armed.next_probe_at, Some(first_deadline));
14208        tokio::time::advance(Duration::from_secs(29)).await;
14209        let turns = actor.turns();
14210        actor
14211            .registry
14212            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14213            .unwrap();
14214        let updated = actor
14215            .wait_for_select(|checkpoint| checkpoint.turn > turns)
14216            .await;
14217        assert_eq!(
14218            updated.next_probe_at,
14219            Some(first_deadline),
14220            "catalog update must not reset cadence"
14221        );
14222        actor.no_probe();
14223        tokio::time::advance(first_deadline - Instant::now()).await;
14224        let frame = actor.probe().await;
14225        actor.answer(1, frame);
14226        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14227        let next_deadline = Instant::now() + next;
14228        let rearmed = actor
14229            .wait_for_select(|checkpoint| {
14230                checkpoint
14231                    .next_probe_at
14232                    .is_some_and(|deadline| deadline > first_deadline)
14233            })
14234            .await;
14235        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14236        tokio::time::advance(next - Duration::from_nanos(1)).await;
14237        actor.no_probe();
14238        tokio::time::advance(Duration::from_nanos(1)).await;
14239        actor.probe().await;
14240    }
14241
14242    #[tokio::test(start_paused = true)]
14243    async fn disconnect_disarms_and_reconnect_rearms_health() {
14244        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14245        actor.hello(1, true);
14246        actor
14247            .wait_for_select(|checkpoint| {
14248                checkpoint.registered_connection == Some(ConnectionId::new(1))
14249                    && checkpoint.next_probe_at.is_some()
14250            })
14251            .await;
14252        let turns = actor.turns();
14253        let tick = Instant::now();
14254        actor
14255            .registry
14256            .deregister_connection(ConnectionId::new(1))
14257            .unwrap();
14258        let disarmed = actor
14259            .wait_for_select(|checkpoint| {
14260                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14261            })
14262            .await;
14263        assert_eq!(disarmed.next_probe_at, None);
14264        assert_eq!(disarmed.wake_after, None);
14265        assert_eq!(
14266            Instant::now(),
14267            tick,
14268            "disconnect must disarm in the notification tick"
14269        );
14270        tokio::time::advance(Duration::from_secs(60)).await;
14271        actor.no_probe();
14272        let reconnect_tick = Instant::now();
14273        actor.hello(2, true);
14274        let rearmed = actor
14275            .wait_for_select(|checkpoint| {
14276                checkpoint.registered_connection == Some(ConnectionId::new(2))
14277                    && checkpoint.next_probe_at.is_some()
14278            })
14279            .await;
14280        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14281        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14282        tokio::time::advance(delay).await;
14283        actor.probe().await;
14284    }
14285
14286    #[tokio::test(start_paused = true)]
14287    async fn reregistration_without_health_disarms_the_old_schedule() {
14288        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14289        actor.hello(1, true);
14290        actor
14291            .wait_for_select(|checkpoint| {
14292                checkpoint.registered_connection == Some(ConnectionId::new(1))
14293                    && checkpoint.next_probe_at.is_some()
14294            })
14295            .await;
14296        actor
14297            .registry
14298            .deregister_connection(ConnectionId::new(1))
14299            .unwrap();
14300        actor.hello(2, false);
14301        let disarmed = actor
14302            .wait_for_select(|checkpoint| {
14303                checkpoint.registered_connection == Some(ConnectionId::new(2))
14304            })
14305            .await;
14306        assert_eq!(disarmed.next_probe_at, None);
14307        assert_eq!(disarmed.wake_after, None);
14308        tokio::time::advance(Duration::from_secs(60)).await;
14309        actor.no_probe();
14310    }
14311
14312    #[tokio::test(start_paused = true)]
14313    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14314        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14315        actor.hello(1, true);
14316        let armed = actor
14317            .wait_for_select(|checkpoint| {
14318                checkpoint.registered_connection == Some(ConnectionId::new(1))
14319                    && checkpoint.next_probe_at.is_some()
14320            })
14321            .await;
14322        // Simulate the old process's connection closing before the restart
14323        // tears the process down. The replacement process has started but has
14324        // not sent its HELLO, so it is not registered.
14325        actor
14326            .registry
14327            .deregister_connection(ConnectionId::new(1))
14328            .unwrap();
14329        actor.module.restart(Some(0)).await.unwrap();
14330        let parked = actor
14331            .wait_for_select(|checkpoint| {
14332                checkpoint.generation > armed.generation
14333                    && checkpoint.registered_connection.is_none()
14334                    && checkpoint.next_probe_at.is_none()
14335            })
14336            .await;
14337        assert_eq!(parked.wake_after, None);
14338        actor.rx.close();
14339        // Drain the stop notice the restart queued for the old process's connection,
14340        // so it isn't mistaken for traffic to the replacement.
14341        while actor.rx.try_recv().is_ok() {}
14342        let turns = actor.turns();
14343        tokio::time::advance(Duration::from_secs(60)).await;
14344        let after = actor.turns();
14345        eprintln!(
14346            "restart park turns: before={turns} after={after} delta={}",
14347            after - turns
14348        );
14349        assert_eq!(after, turns, "replacement must park before HELLO");
14350        let hello_tick = Instant::now();
14351        actor.hello(2, true);
14352        let rearmed = actor
14353            .wait_for_select(|checkpoint| {
14354                checkpoint.registered_connection == Some(ConnectionId::new(2))
14355                    && checkpoint.next_probe_at.is_some()
14356            })
14357            .await;
14358        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14359        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14360        tokio::time::advance(delay).await;
14361        actor.probe().await;
14362    }
14363
14364    #[tokio::test(start_paused = true)]
14365    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14366        use tokio::{
14367            io::{AsyncReadExt, AsyncWriteExt},
14368            net::TcpListener,
14369        };
14370        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14371        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14372        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14373        let (started_tx, mut started_rx) = mpsc::channel(8);
14374        let peer = tokio::spawn(async move {
14375            loop {
14376                let (mut socket, _) = listener.accept().await.unwrap();
14377                let mut request = [0; 1024];
14378                assert!(socket.read(&mut request).await.unwrap() > 0);
14379                started_tx.send(Instant::now()).await.unwrap();
14380                socket
14381                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14382                    .await
14383                    .unwrap();
14384            }
14385        });
14386        actor
14387            .module
14388            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14389            .await
14390            .unwrap();
14391        let first_tick = Instant::now();
14392        let first = actor
14393            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14394            .await;
14395        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14396        assert!(started_rx.try_recv().is_err());
14397
14398        // Restart just before the old deadline: reusing it would give the new
14399        // process only one second to start, rather than a full cadence.
14400        tokio::time::advance(
14401            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14402        )
14403        .await;
14404        actor.module.restart(Some(0)).await.unwrap();
14405        let spawn_tick = Instant::now();
14406        let replacement = actor
14407            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14408            .await;
14409        let deadline = replacement.next_probe_at.unwrap();
14410        assert!(
14411            deadline >= spawn_tick + actor.health.cadence,
14412            "a replacement must not inherit its predecessor's probe deadline"
14413        );
14414        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14415        assert!(
14416            started_rx.try_recv().is_err(),
14417            "no HTTP probe during startup grace"
14418        );
14419        tokio::time::advance(deadline - Instant::now()).await;
14420        let landed = wait_for("first replacement HTTP probe", || {
14421            started_rx.try_recv().ok()
14422        })
14423        .await;
14424        assert!(landed >= spawn_tick + actor.health.cadence);
14425        wait_for("healthy replacement", || {
14426            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14427                .then_some(())
14428        })
14429        .await;
14430        peer.abort();
14431    }
14432
14433    #[tokio::test(start_paused = true)]
14434    async fn rescan_adds_and_removes_http_health_without_polling() {
14435        use tokio::{
14436            io::{AsyncReadExt, AsyncWriteExt},
14437            net::TcpListener,
14438        };
14439        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14440        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14441        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14442        // One millisecond separates successive probes without a polling delay.
14443        actor.health.cadence = Duration::from_millis(1);
14444        let (started_tx, mut started_rx) = mpsc::channel(8);
14445        let peer = tokio::spawn(async move {
14446            loop {
14447                let (mut socket, _) = listener.accept().await.unwrap();
14448                let mut request = [0; 1024];
14449                assert!(socket.read(&mut request).await.unwrap() > 0);
14450                started_tx.send(()).await.unwrap();
14451                socket
14452                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14453                    .await
14454                    .unwrap();
14455            }
14456        });
14457        let tick = Instant::now();
14458        let turns = actor.turns();
14459        actor
14460            .module
14461            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14462            .await
14463            .unwrap();
14464        let armed = actor
14465            .wait_for_select(|checkpoint| {
14466                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14467            })
14468            .await;
14469        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14470        assert_eq!(Instant::now(), tick);
14471        tokio::time::advance(Duration::from_millis(1)).await;
14472        let mut started = false;
14473        wait_for("successful HTTP probe", || {
14474            started |= started_rx.try_recv().is_ok();
14475            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14476                .then_some(())
14477        })
14478        .await;
14479        assert!(started, "added HTTP check must start at its first deadline");
14480        assert_eq!(
14481            actor.module.status().unwrap().health.status,
14482            SupervisorHealthStatus::Ok
14483        );
14484        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14485        actor.health.http = None;
14486        let turns = actor.turns();
14487        actor
14488            .module
14489            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14490            .await
14491            .unwrap();
14492        let parked = actor
14493            .wait_for_select(|checkpoint| {
14494                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14495            })
14496            .await;
14497        assert_eq!(parked.wake_after, None);
14498        let turns = actor.turns();
14499        tokio::time::advance(Duration::from_secs(60)).await;
14500        assert_eq!(
14501            actor.turns(),
14502            turns,
14503            "removed HTTP check must park the actor"
14504        );
14505        assert!(
14506            started_rx.try_recv().is_err(),
14507            "removed HTTP check must stay disarmed"
14508        );
14509        assert_eq!(
14510            actor.module.status().unwrap().health,
14511            ModuleHealthStatus::default()
14512        );
14513        peer.abort();
14514    }
14515}
14516
14517#[cfg(test)]
14518mod drain_busy_reprobe_tests {
14519    //! The drain's busy-gauge wait, driven against an in-process module
14520    //! connection whose health answers the test scripts. Time is paused, so
14521    //! every elapsed duration below is exact virtual time.
14522    use super::*;
14523    use std::collections::BTreeMap;
14524    use std::sync::Mutex as StdMutex;
14525    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14526
14527    const MODULE_ID: &str = "drain-reprobe-module";
14528    const CONNECTION: u64 = 7;
14529    const GAUGE: &str = "runs_in_flight";
14530    /// The production default for both the health cadence and the drain
14531    /// ceiling, which is the pairing that used to cost a full ceiling.
14532    const CADENCE: Duration = Duration::from_secs(30);
14533    const CEILING: Duration = Duration::from_secs(30);
14534
14535    /// How the scripted module answers each `health.check`.
14536    #[derive(Clone, Copy)]
14537    enum Answer {
14538        /// Report the gauge busy until this instant, quiet from then on, after
14539        /// taking `latency` to answer.
14540        BusyUntil { until: Instant, latency: Duration },
14541        /// Answer at once, with metrics that leave the declared gauge out.
14542        OmitsGauge,
14543        /// Never answer.
14544        Silent,
14545    }
14546
14547    struct Drain {
14548        forwarding: Arc<ForwardingTable>,
14549        runtime: SupervisorRuntimeConfig,
14550        endpoint: crate::ModuleEndpointId,
14551        /// When each probe reached the module.
14552        probes: Arc<StdMutex<Vec<Instant>>>,
14553        responder: tokio::task::JoinHandle<()>,
14554    }
14555
14556    impl Drain {
14557        fn start(answer: Answer) -> Self {
14558            let registry = Arc::new(Registry::default());
14559            let forwarding = Arc::new(ForwardingTable::default());
14560            let health = HealthConfig {
14561                cadence: CADENCE,
14562                ..HealthConfig::default()
14563            };
14564            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14565                .with_forwarding(forwarding.clone())
14566                .with_health_config(health)
14567                .runtime_config();
14568            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14569            let endpoint = forwarding
14570                .register_module_connection(
14571                    ConnectionId::new(CONNECTION),
14572                    MODULE_ID.to_string(),
14573                    subc_protocol::PROTOCOL_VERSION,
14574                    Concurrency::ModuleManaged,
14575                    FrameSink::new(tx),
14576                )
14577                .unwrap();
14578            let probes = Arc::new(StdMutex::new(Vec::new()));
14579            let responder = tokio::spawn({
14580                let forwarding = forwarding.clone();
14581                let probes = probes.clone();
14582                async move {
14583                    while let Some(frame) = rx.recv().await {
14584                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14585                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14586                        probes.lock().unwrap().push(Instant::now());
14587                        let metrics = match answer {
14588                            Answer::BusyUntil { until, latency } => {
14589                                sleep(latency).await;
14590                                let busy = u64::from(Instant::now() < until);
14591                                serde_json::json!({ GAUGE: busy })
14592                            }
14593                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14594                            Answer::Silent => continue,
14595                        };
14596                        // A probe abandoned at its deadline has no RPC left to
14597                        // complete, so a late answer may be refused.
14598                        let _ = forwarding.complete_module_control_rpc(
14599                            ConnectionId::new(CONNECTION),
14600                            frame.header.corr,
14601                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14602                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14603                                status: HealthStatus::Ok,
14604                                detail: None,
14605                                metrics: Some(metrics),
14606                            }),
14607                        );
14608                    }
14609                }
14610            });
14611            Self {
14612                forwarding,
14613                runtime,
14614                endpoint,
14615                probes,
14616                responder,
14617            }
14618        }
14619
14620        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14621        /// how long it took.
14622        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14623            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14624            (drained, elapsed)
14625        }
14626
14627        /// [`Self::run`], also returning what the drain's probes read.
14628        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14629            let started = Instant::now();
14630            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14631            let drained = wait_for_forwarding_quiescence(
14632                &self.forwarding,
14633                MODULE_ID,
14634                &self.runtime,
14635                self.endpoint,
14636                started + ceiling,
14637                &[GAUGE.to_string()],
14638                DrainScope::Active,
14639                &mut trace,
14640            )
14641            .await
14642            .unwrap();
14643            (drained, started.elapsed(), trace)
14644        }
14645
14646        fn probes(&self) -> Vec<Instant> {
14647            self.probes.lock().unwrap().clone()
14648        }
14649    }
14650
14651    impl Drop for Drain {
14652        fn drop(&mut self) {
14653            self.responder.abort();
14654        }
14655    }
14656
14657    #[tokio::test(start_paused = true)]
14658    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14659        let drain = Drain::start(Answer::BusyUntil {
14660            until: Instant::now() + Duration::from_secs(1),
14661            latency: Duration::ZERO,
14662        });
14663        let (drained, elapsed) = drain.run(CEILING).await;
14664        assert!(
14665            drained,
14666            "gauges that read zero during the drain must let it finish drained, \
14667             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14668        );
14669        assert!(
14670            elapsed < Duration::from_secs(3),
14671            "drain took {elapsed:?} after the gauges went quiet at 1s"
14672        );
14673        let probes = drain.probes();
14674        assert!(
14675            probes.len() >= 2,
14676            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14677        );
14678    }
14679
14680    /// The same shape in real time on a multi-threaded runtime: the first
14681    /// probe is answered busy at once, within a millisecond of the drain
14682    /// starting, and later probes read idle. The drain must keep probing
14683    /// after that first instant answer and end soon after the gauge falls.
14684    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14685    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14686        let drain = Drain::start(Answer::BusyUntil {
14687            until: Instant::now() + Duration::from_millis(600),
14688            latency: Duration::ZERO,
14689        });
14690        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14691        assert!(drained, "the gauge fell at 600ms: {trace}");
14692        assert!(
14693            elapsed < Duration::from_secs(3),
14694            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14695        );
14696        let probes = drain.probes();
14697        assert!(
14698            probes.len() >= 2 && trace.probes as usize == probes.len(),
14699            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14700        );
14701    }
14702
14703    #[tokio::test(start_paused = true)]
14704    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14705        let started = Instant::now();
14706        let drain = Drain::start(Answer::BusyUntil {
14707            until: started + Duration::from_secs(3600),
14708            latency: Duration::ZERO,
14709        });
14710        let (drained, elapsed) = drain.run(CEILING).await;
14711        assert!(!drained, "a module busy throughout must not drain");
14712        assert!(
14713            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14714            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14715        );
14716        // Re-probing is paced by the interval, not a busy loop: at most one
14717        // probe per interval across the ceiling, plus the first at time zero.
14718        let probes = drain.probes();
14719        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14720        assert!(
14721            probes.len() as u128 <= max_probes,
14722            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14723            probes.len()
14724        );
14725        for pair in probes.windows(2) {
14726            assert!(
14727                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14728                "consecutive probes {:?} apart",
14729                pair[1] - pair[0]
14730            );
14731        }
14732    }
14733
14734    #[tokio::test(start_paused = true)]
14735    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14736        // Each answer takes longer than the re-probe interval; the next probe
14737        // must wait for the previous answer and then the interval.
14738        let latency = Duration::from_secs(1);
14739        let drain = Drain::start(Answer::BusyUntil {
14740            until: Instant::now() + Duration::from_secs(3600),
14741            latency,
14742        });
14743        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14744        assert!(!drained);
14745        let probes = drain.probes();
14746        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14747        for pair in probes.windows(2) {
14748            assert!(
14749                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14750                "a probe started {:?} after the previous one, before it could finish",
14751                pair[1] - pair[0]
14752            );
14753        }
14754    }
14755
14756    #[tokio::test(start_paused = true)]
14757    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14758        let drain = Drain::start(Answer::Silent);
14759        let ceiling = Duration::from_secs(2);
14760        let (drained, elapsed) = drain.run(ceiling).await;
14761        assert!(!drained, "an unanswered probe counts as busy");
14762        assert!(
14763            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14764            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14765        );
14766        assert_eq!(
14767            drain.probes().len(),
14768            1,
14769            "the unanswered probe holds the only slot until the ceiling"
14770        );
14771    }
14772
14773    /// Every event logged while it is installed, as `message` plus fields.
14774    #[derive(Clone, Default)]
14775    struct LogCapture {
14776        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14777    }
14778
14779    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14780        fn on_event(
14781            &self,
14782            event: &tracing::Event<'_>,
14783            _context: tracing_subscriber::layer::Context<'_, S>,
14784        ) {
14785            struct Fields(BTreeMap<String, String>);
14786            impl tracing::field::Visit for Fields {
14787                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14788                    self.0
14789                        .insert(field.name().to_string(), format!("{value:?}"));
14790                }
14791            }
14792            let mut fields = Fields(BTreeMap::new());
14793            event.record(&mut fields);
14794            self.events.lock().unwrap().push(fields.0);
14795        }
14796    }
14797
14798    impl LogCapture {
14799        fn install(&self) -> tracing::subscriber::DefaultGuard {
14800            use tracing_subscriber::layer::SubscriberExt;
14801            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14802        }
14803
14804        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14805            self.events
14806                .lock()
14807                .unwrap()
14808                .iter()
14809                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14810                .cloned()
14811                .collect()
14812        }
14813    }
14814
14815    /// A module that declares a busy gauge but leaves it out of its health
14816    /// answers holds a drain to its ceiling. The timeout line must say so by
14817    /// name, and the drain must warn about the missing gauge once, not on
14818    /// every probe.
14819    #[tokio::test(start_paused = true)]
14820    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14821        let capture = LogCapture::default();
14822        let _guard = capture.install();
14823        let drain = Drain::start(Answer::OmitsGauge);
14824        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14825        assert!(!drained, "a gauge never reported cannot read quiet");
14826        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14827
14828        let missing = capture.with_message("lacks a declared busy gauge");
14829        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14830        assert_eq!(
14831            missing[0].get("missing_gauges").map(String::as_str),
14832            Some(GAUGE)
14833        );
14834
14835        warn_drain_timed_out(
14836            MODULE_ID,
14837            Duration::from_secs(2),
14838            RouteCloseReason::Restart,
14839            &Default::default(),
14840            &trace,
14841        );
14842        let timeouts = capture.with_message("route drain timed out");
14843        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14844        let gauges = &timeouts[0]["busy_gauges"];
14845        assert!(
14846            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14847            "the timeout line must name the gauge and why it read busy: {gauges}"
14848        );
14849        assert!(
14850            gauges.ends_with("; no value ever read"),
14851            "no value was ever read for the gauge: {gauges}"
14852        );
14853        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14854    }
14855
14856    /// The trace names a busy value and when it was read. For a probe that
14857    /// never got an answer, it names the probe's error kind and says that no
14858    /// value was ever read.
14859    #[tokio::test(start_paused = true)]
14860    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14861        let busy = Drain::start(Answer::BusyUntil {
14862            until: Instant::now() + Duration::from_secs(3600),
14863            latency: Duration::ZERO,
14864        });
14865        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14866        let rendered = trace.to_string();
14867        assert!(
14868            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14869                && rendered.contains("; last value non_zero(1) @"),
14870            "{rendered}"
14871        );
14872
14873        let silent = Drain::start(Answer::Silent);
14874        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14875        // The probe is capped at the 1 s ceiling, and its reading is stamped
14876        // when it gave up.
14877        assert_eq!(
14878            trace.to_string(),
14879            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14880        );
14881        assert_eq!(trace.probes, 1);
14882    }
14883
14884    /// A gauge value that is not a count reads busy and is shown as sent.
14885    #[test]
14886    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14887        let gauges = [GAUGE.to_string()];
14888        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14889        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14890        assert_eq!(
14891            busy_gauge_observation(&verdicts),
14892            BusyGaugeObservation::Busy
14893        );
14894        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14895        assert_eq!(
14896            busy_gauge_observation(&zero),
14897            BusyGaugeObservation::Quiescent
14898        );
14899        let none = gauge_verdicts(None, &gauges);
14900        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14901    }
14902}
14903
14904#[cfg(test)]
14905mod health_evidence_tests {
14906    use super::{HealthProbeError, HealthProbeEvidence};
14907    use std::collections::HashSet;
14908
14909    /// The evidential asymmetry, asserted rather than described.
14910    ///
14911    /// Exactly ONE observation is proof a module cannot serve, and the one that
14912    /// fires under CPU starvation is not it. Before the split, all fifteen
14913    /// construction sites collapsed into a single String, so a timeout carried the
14914    /// same weight as a dead lane -- which is how a healthy module was restarted
14915    /// three times in one day.
14916    #[test]
14917    fn only_a_dead_lane_is_proof_of_death() {
14918        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14919        // Three non-proof classes, each for a different reason: silence is
14920        // consistent with health, a bad answer proves the module ALIVE, and a
14921        // daemon-side fault never reached the module at all.
14922        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14923        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14924        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14925    }
14926
14927    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14928    ///
14929    /// A shared label renders two different observations identically in the line an
14930    /// operator reads after an unexplained restart -- the exact confusion this
14931    /// change removes.
14932    #[test]
14933    fn every_evidence_class_has_a_distinct_label() {
14934        let labels = [
14935            HealthProbeError::lane_dead("").label(),
14936            HealthProbeError::no_answer("").label(),
14937            HealthProbeError::bad_answer("").label(),
14938            HealthProbeError::misconfigured("").label(),
14939        ];
14940        let unique: HashSet<_> = labels.iter().collect();
14941        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14942    }
14943
14944    /// The class is additional information, not a replacement.
14945    ///
14946    /// An operator needs both "this was silence" and the specific text saying how
14947    /// long we waited; a classification that swallowed the message would trade one
14948    /// missing distinction for another.
14949    #[test]
14950    fn classification_preserves_the_original_message() {
14951        let err = HealthProbeError::no_answer("module did not answer within 5s");
14952        assert_eq!(err.to_string(), "module did not answer within 5s");
14953        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14954    }
14955}
14956
14957#[cfg(test)]
14958mod health_tombstone_tests {
14959    use std::{path::PathBuf, sync::Arc, time::Duration};
14960
14961    use subc_protocol::{
14962        manifest::Concurrency,
14963        session::{HealthStatus, ModuleControlResponse},
14964    };
14965    use tokio::sync::mpsc;
14966
14967    use super::{
14968        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14969        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14970    };
14971    use crate::{
14972        control::ControlHandler,
14973        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14974        registry::{ConnectionId, Registry},
14975        router::FrameSink,
14976    };
14977
14978    struct ProbeHarness {
14979        spec: ModuleSpec,
14980        runtime: SupervisorRuntimeConfig,
14981        forwarding: Arc<ForwardingTable>,
14982        module_connection: ConnectionId,
14983        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14984        handler: ControlHandler,
14985        module: super::SupervisedModule,
14986    }
14987
14988    fn probe_harness() -> ProbeHarness {
14989        let registry = Arc::new(Registry::default());
14990        let forwarding = Arc::new(ForwardingTable::default());
14991        let supervisor_handle = super::SupervisorHandle::new();
14992        let health = HealthConfig {
14993            http: None,
14994            cadence: Duration::from_secs(30),
14995            deadline: Duration::from_secs(5),
14996            failure_threshold: 3,
14997            on_degraded: HealthAction::Report,
14998            on_failing: HealthAction::Report,
14999            critical: false,
15000        };
15001        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15002            .with_forwarding(Arc::clone(&forwarding))
15003            .with_handle(supervisor_handle.clone())
15004            .with_health_config(health);
15005        let spec = ModuleSpec {
15006            module_id: "late-health-module".to_string(),
15007            program: PathBuf::from("disabled-module"),
15008            args: Vec::new(),
15009            env: Vec::new(),
15010            reserved: false,
15011            reserved_prefixes: Vec::new(),
15012            protocol: ModuleProtocol::Subc,
15013            overlap: Default::default(),
15014        };
15015        let module = supervisor
15016            .supervise_configured(spec.clone(), false)
15017            .unwrap();
15018        let runtime = supervisor.runtime_config();
15019        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15020            .with_supervisor(supervisor_handle);
15021        let module_connection = ConnectionId::new(700);
15022        let (module_tx, module_rx) = mpsc::channel(8);
15023        forwarding
15024            .register_module_connection(
15025                module_connection,
15026                spec.module_id.clone(),
15027                subc_protocol::PROTOCOL_VERSION,
15028                Concurrency::ModuleManaged,
15029                FrameSink::new(module_tx),
15030            )
15031            .unwrap();
15032
15033        ProbeHarness {
15034            spec,
15035            runtime,
15036            forwarding,
15037            module_connection,
15038            module_rx,
15039            handler,
15040            module,
15041        }
15042    }
15043
15044    async fn finish_after(
15045        harness: &mut ProbeHarness,
15046        stall: Duration,
15047    ) -> ModuleControlRpcCompletion {
15048        assert!(stall > harness.runtime.health.deadline);
15049        let deadline = harness.runtime.health.deadline;
15050        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15051        let answer = async {
15052            let frame = harness.module_rx.recv().await.expect("health.check frame");
15053            tokio::time::advance(deadline).await;
15054            tokio::task::yield_now().await;
15055            tokio::time::advance(stall - deadline).await;
15056            harness
15057                .forwarding
15058                .complete_module_control_rpc(
15059                    harness.module_connection,
15060                    frame.header.corr,
15061                    Some("health.check"),
15062                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15063                        status: HealthStatus::Ok,
15064                        detail: None,
15065                        metrics: None,
15066                    }),
15067                )
15068                .unwrap()
15069        };
15070        let (probe_result, completion) = tokio::join!(probe, answer);
15071        let err = probe_result.expect_err("probe must miss its deadline");
15072        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15073        completion
15074    }
15075
15076    async fn time_out_without_answer(harness: &mut ProbeHarness) {
15077        let deadline = harness.runtime.health.deadline;
15078        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15079        let exhaust_deadline = async {
15080            let _frame = harness.module_rx.recv().await.expect("health.check frame");
15081            tokio::time::advance(deadline).await;
15082            tokio::task::yield_now().await;
15083        };
15084        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15085        let err = probe_result.expect_err("probe must miss its deadline");
15086        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15087    }
15088
15089    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15090        let registry = Arc::clone(&harness.module.inner.registry);
15091        let snapshot = Arc::clone(&harness.module.inner.snapshot);
15092        let process_liveness = super::SupervisorProcessLiveness::default();
15093        let mut child = None;
15094        let cycle = super::run_health_probe_cycle(
15095            &harness.spec,
15096            &harness.runtime,
15097            &registry,
15098            &process_liveness,
15099            &snapshot,
15100            &mut child,
15101        );
15102        let peer = async {
15103            let frame = harness.module_rx.recv().await.expect("health.check frame");
15104            if answer {
15105                harness
15106                    .forwarding
15107                    .complete_module_control_rpc(
15108                        harness.module_connection,
15109                        frame.header.corr,
15110                        Some("health.check"),
15111                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15112                            status: HealthStatus::Ok,
15113                            detail: None,
15114                            metrics: Some(serde_json::json!({"ready": true})),
15115                        }),
15116                    )
15117                    .unwrap();
15118            } else {
15119                tokio::time::advance(harness.runtime.health.deadline).await;
15120                tokio::task::yield_now().await;
15121            }
15122        };
15123        tokio::join!(cycle, peer);
15124    }
15125
15126    #[tokio::test(start_paused = true)]
15127    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15128        let mut harness = probe_harness();
15129        // Drive the probe cycle directly with an in-memory wire peer. Stop the
15130        // disabled module's monitor so only this test owns lifecycle transitions;
15131        // no OS process is launched, and a restart is observed at scheduling.
15132        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15133        monitor.abort();
15134        let _ = monitor.await;
15135        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15136            state.enabled = true;
15137            state.state = super::ModuleState::Running;
15138            state.process_alive = true;
15139        })
15140        .unwrap();
15141
15142        run_probe_cycle(&mut harness, true).await;
15143        assert_eq!(
15144            harness.module.status().unwrap().health.status,
15145            super::SupervisorHealthStatus::Ok
15146        );
15147
15148        for failures in 1..harness.runtime.health.failure_threshold {
15149            run_probe_cycle(&mut harness, false).await;
15150            let status = harness.module.status().unwrap();
15151            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15152            assert_eq!(status.health.consecutive_failures, failures);
15153            assert!(status.health.last_probe_ms.is_some());
15154            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15155            assert!(status.health.metrics.is_none());
15156            assert_eq!(status.state, super::ModuleState::Running);
15157            assert!(status.process_alive);
15158            assert_eq!(status.restart_count, 0);
15159            assert_eq!(status.lifetime_restarts, 0);
15160            assert!(status.health.last_action.is_none());
15161            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15162        }
15163
15164        run_probe_cycle(&mut harness, true).await;
15165        let recovered = harness.module.status().unwrap();
15166        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15167        assert_eq!(recovered.health.consecutive_failures, 0);
15168        assert!(recovered.health.detail.is_none());
15169        assert_eq!(
15170            recovered.health.metrics,
15171            Some(serde_json::json!({"ready": true}))
15172        );
15173        assert_eq!(recovered.lifetime_restarts, 0);
15174
15175        for failures in 1..=harness.runtime.health.failure_threshold {
15176            run_probe_cycle(&mut harness, false).await;
15177            let status = harness.module.status().unwrap();
15178            assert_eq!(status.health.consecutive_failures, failures);
15179            if failures < harness.runtime.health.failure_threshold {
15180                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15181                assert_eq!(status.state, super::ModuleState::Running);
15182                assert_eq!(status.lifetime_restarts, 0);
15183                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15184            } else {
15185                assert_eq!(
15186                    status.health.status,
15187                    super::SupervisorHealthStatus::Unresponsive
15188                );
15189                assert_eq!(status.state, super::ModuleState::Restarting);
15190                assert_eq!(status.restart_count, 1);
15191                assert_eq!(status.lifetime_restarts, 1);
15192                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15193                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15194            }
15195        }
15196    }
15197
15198    #[tokio::test(start_paused = true)]
15199    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15200        let mut harness = probe_harness();
15201
15202        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15203        let first_latency = match &first {
15204            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15205            other => panic!("late answer was not retained: {other:?}"),
15206        };
15207        assert!(harness.handler.observe_module_control_completion(first));
15208
15209        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15210        let second_latency = match &second {
15211            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15212            other => panic!("late answer was not retained: {other:?}"),
15213        };
15214        assert!(harness.handler.observe_module_control_completion(second));
15215
15216        assert_eq!(first_latency, Duration::from_secs(8));
15217        assert_eq!(
15218            second_latency - first_latency,
15219            Duration::from_secs(3),
15220            "latency must grow linearly with the additional stall"
15221        );
15222        let health = harness.module.status().unwrap().health;
15223        assert_eq!(health.late_answer_count, 2);
15224        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15225    }
15226
15227    /// A module that answers every probe late must never march to the kill
15228    /// threshold: the late answer proves it is alive, so it must clear the miss
15229    /// streak the timeout recorded. Without the reset, a CPU-starved module
15230    /// that serves every probe seconds past the deadline accumulates
15231    /// `consecutive_failures` to the threshold and is killed — the exact
15232    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15233    /// "proves the module is alive" five times while counting five misses.
15234    #[tokio::test(start_paused = true)]
15235    async fn late_answer_clears_the_consecutive_failure_streak() {
15236        let mut harness = probe_harness();
15237
15238        // Timeout recorded first: the probe path saw no answer in time.
15239        time_out_without_answer(&mut harness).await;
15240        harness
15241            .module
15242            .record_health_probe_failure_for_test("[no-answer] test miss")
15243            .unwrap();
15244        assert_eq!(
15245            harness.module.status().unwrap().health.consecutive_failures,
15246            1,
15247            "precondition: the miss must be on the streak before the late answer"
15248        );
15249
15250        // The stalled reply then lands: proof of life.
15251        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15252        assert!(matches!(
15253            late,
15254            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15255        ));
15256        assert!(harness.handler.observe_module_control_completion(late));
15257
15258        let health = harness.module.status().unwrap().health;
15259        assert_eq!(
15260            health.consecutive_failures, 0,
15261            "a late answer is an answer: the streak must reset"
15262        );
15263        assert_eq!(health.late_answer_count, 1);
15264    }
15265
15266    #[tokio::test(start_paused = true)]
15267    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15268        let mut harness = probe_harness();
15269
15270        for _ in 0..20 {
15271            time_out_without_answer(&mut harness).await;
15272            assert_eq!(
15273                harness.forwarding.health_probe_tombstone_count().unwrap(),
15274                1
15275            );
15276        }
15277    }
15278}
15279
15280#[cfg(test)]
15281mod child_env_tests {
15282    #[cfg(windows)]
15283    use super::{
15284        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15285    };
15286    #[cfg(windows)]
15287    use std::sync::{Arc, OnceLock};
15288    #[cfg(windows)]
15289    #[test]
15290    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15291        let spec = spec(Vec::new());
15292        let mut command = Command::new("unused");
15293        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15294            .unwrap()
15295            .unwrap();
15296        let name = command
15297            .as_std()
15298            .get_envs()
15299            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15300            .and_then(|(_, value)| value)
15301            .unwrap();
15302        assert_eq!(name, OsStr::new(handoff.name()));
15303        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15304        assert!(command.as_std().get_envs().any(|(key, value)| key
15305            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15306            && value == Some(OsStr::new("env"))));
15307        assert!(command
15308            .as_std()
15309            .get_envs()
15310            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15311    }
15312
15313    #[cfg(windows)]
15314    #[test]
15315    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15316        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15317        snapshot.process_alive = true;
15318        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15319        assert_eq!(
15320            observed_launch_nonce_source(&snapshot).as_deref(),
15321            Some("env")
15322        );
15323        let consumed = Arc::new(OnceLock::new());
15324        snapshot.pipe_consumed = Some(consumed.clone());
15325        assert_eq!(
15326            observed_launch_nonce_source(&snapshot).as_deref(),
15327            Some("env")
15328        );
15329        consumed.set(()).unwrap();
15330        assert_eq!(
15331            observed_launch_nonce_source(&snapshot).as_deref(),
15332            Some("pipe")
15333        );
15334        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15335        assert_eq!(
15336            observed_launch_nonce_source(&snapshot).as_deref(),
15337            Some("env")
15338        );
15339        clear_current_process_facts(&mut snapshot);
15340        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15341    }
15342    use super::{
15343        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15344        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15345        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15346    };
15347    use std::{ffi::OsStr, path::PathBuf};
15348    use tokio::process::Command;
15349
15350    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15351        ModuleSpec {
15352            module_id: "env-plan".to_string(),
15353            program: PathBuf::from("/nonexistent"),
15354            args: Vec::new(),
15355            env,
15356            reserved: false,
15357            reserved_prefixes: Vec::new(),
15358            protocol: ModuleProtocol::Subc,
15359            overlap: Default::default(),
15360        }
15361    }
15362
15363    #[test]
15364    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15365        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15366        let digest = LaunchSpecDigest::from_spec(&base);
15367        let mut reordered = base.clone();
15368        reordered.env.reverse();
15369        reordered.module_id = "different-id".into();
15370        reordered.reserved = true;
15371        reordered.overlap = Default::default();
15372        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15373        let mut delimiter_collision = base.clone();
15374        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15375        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15376        let mut changed = base.clone();
15377        changed.protocol = ModuleProtocol::None;
15378        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15379        changed = base.clone();
15380        changed.args.push("--flag".into());
15381        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15382        changed = base.clone();
15383        changed.program = PathBuf::from("/other");
15384        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15385        changed = base.clone();
15386        changed.env[1].1 = "other".into();
15387        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15388        assert!(!format!("{digest:?}").contains("B=C"));
15389        assert!(!format!("{digest:?}").contains("[u8"));
15390    }
15391
15392    #[cfg(unix)]
15393    #[test]
15394    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15395        use std::os::unix::ffi::OsStringExt;
15396        let mut left = spec(Vec::new());
15397        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15398        let mut right = left.clone();
15399        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15400        assert_eq!(
15401            left.program.to_string_lossy(),
15402            right.program.to_string_lossy()
15403        );
15404        assert_ne!(
15405            LaunchSpecDigest::from_spec(&left),
15406            LaunchSpecDigest::from_spec(&right)
15407        );
15408    }
15409
15410    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
15411    /// one still gets its own.
15412    ///
15413    /// This is the narrow goal `env_clear()` was reached for, and the reason the
15414    /// fix is `env_remove` rather than deleting the line: an operator's ambient
15415    /// filter silently becoming an unconfigured module's log level is a real
15416    /// defect, just a much smaller one than clearing the environment.
15417    ///
15418    /// Asserted on the command plan rather than a spawned child because proving
15419    /// the ABSENCE of an inherited variable needs the parent's environment
15420    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
15421    /// removal as `(key, None)`, which is exactly the distinction wanted: not
15422    /// "absent because nobody set it" but "explicitly unset for the child".
15423    #[test]
15424    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15425        let mut command = Command::new("/nonexistent");
15426        apply_child_env(&mut command, &spec(Vec::new()));
15427        let removed = command
15428            .as_std()
15429            .get_envs()
15430            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15431        assert!(
15432            removed,
15433            "ambient CK_LOG must be explicitly removed for an unconfigured module"
15434        );
15435
15436        let mut configured = Command::new("/nonexistent");
15437        apply_child_env(
15438            &mut configured,
15439            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15440        );
15441        let effective = configured
15442            .as_std()
15443            .get_envs()
15444            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15445            .last()
15446            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15447        assert_eq!(
15448            effective,
15449            Some(Some("debug".to_string())),
15450            "a module's configured CK_LOG must survive the ambient removal"
15451        );
15452    }
15453
15454    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
15455    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
15456    /// the same reason as the CK_LOG test above.
15457    ///
15458    /// The argument is the load-bearing half: a stock binary exits on an
15459    /// unknown flag before it listens, so with `--subc` appended the mode
15460    /// could not supervise the one process it exists for. Found by the first
15461    /// conformance run (nats-server: `flag provided but not defined: -subc`).
15462    #[test]
15463    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15464        let connection_file = std::path::Path::new("/run/subc-connection.json");
15465        let handle = SupervisorHandle::new();
15466
15467        let mut none_spec = spec(Vec::new());
15468        none_spec.protocol = ModuleProtocol::None;
15469        let mut none = Command::new("/nonexistent");
15470        let none_handoff =
15471            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15472                .expect("protocol-none spawn args apply");
15473        assert!(
15474            none_handoff.is_none(),
15475            "protocol:none spawn must not receive a nonce descriptor"
15476        );
15477        #[cfg(windows)]
15478        assert!(!none.as_std().get_envs().any(|(key, value)| key
15479            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15480            && value.is_some()));
15481        assert!(
15482            !none.as_std().get_envs().any(|(key, value)| key
15483                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15484                && value.is_some()),
15485            "protocol:none spawn must not name a nonce descriptor"
15486        );
15487        let none_args: Vec<String> = none
15488            .as_std()
15489            .get_args()
15490            .map(|a| a.to_string_lossy().into_owned())
15491            .collect();
15492        assert!(
15493            !none_args.iter().any(|a| a == SUBC_ARG),
15494            "protocol:none argv must not carry --subc; got {none_args:?}"
15495        );
15496        let none_has_nonce = none
15497            .as_std()
15498            .get_envs()
15499            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15500        assert!(
15501            !none_has_nonce,
15502            "protocol:none spawn must not receive a launch nonce"
15503        );
15504        let none_has_module_id = none
15505            .as_std()
15506            .get_envs()
15507            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15508        assert!(
15509            none_has_module_id,
15510            "SUBC_MODULE_ID is inert and stays on every path"
15511        );
15512        assert!(
15513            handle.spawn_nonce(&none_spec.module_id).is_none(),
15514            "no nonce record for a process that will never present one"
15515        );
15516
15517        // Control: the subc-wire path is unchanged by the branch above.
15518        let wire_spec = spec(Vec::new());
15519        let mut wire = Command::new("/nonexistent");
15520        let wire_handoff =
15521            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15522                .expect("subc-wire spawn args apply");
15523        let wire_fd_env = wire
15524            .as_std()
15525            .get_envs()
15526            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15527            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15528        #[cfg(unix)]
15529        assert_eq!(
15530            wire_fd_env,
15531            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15532            "a subc-wire spawn names the pipe it will receive at descriptor 3"
15533        );
15534        #[cfg(windows)]
15535        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15536        #[cfg(not(any(unix, windows)))]
15537        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15538        let wire_args: Vec<String> = wire
15539            .as_std()
15540            .get_args()
15541            .map(|a| a.to_string_lossy().into_owned())
15542            .collect();
15543        assert_eq!(
15544            wire_args,
15545            vec![
15546                SUBC_ARG.to_string(),
15547                connection_file.to_string_lossy().into_owned()
15548            ],
15549            "a subc-wire spawn still carries --subc <path>"
15550        );
15551        assert_eq!(
15552            wire.as_std()
15553                .get_envs()
15554                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15555            !cfg!(unix),
15556            "only Windows supplies the environment nonce"
15557        );
15558        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15559    }
15560
15561    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
15562    /// spec tries to set it; only a swap candidate carries it.
15563    ///
15564    /// "Set it only on candidates" is not enough, because spawn applies the
15565    /// spec's env verbatim and the daemon's own environment is inherited: either
15566    /// could hand a plain restart the swap role, and a module reading it would
15567    /// warm on its long swap budget while callers wait. Asserted as an explicit
15568    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
15569    /// test above gives.
15570    #[test]
15571    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15572        let role = |command: &Command| {
15573            command
15574                .as_std()
15575                .get_envs()
15576                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15577                .last()
15578                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15579        };
15580        let forged = spec(vec![(
15581            SUBC_SPAWN_ROLE_ENV.to_string(),
15582            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15583        )]);
15584
15585        let mut plain = Command::new("/nonexistent");
15586        apply_child_env(&mut plain, &forged);
15587        apply_spawn_role(&mut plain, SpawnRole::Plain);
15588        assert_eq!(
15589            role(&plain),
15590            Some(None),
15591            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15592        );
15593
15594        let mut candidate = Command::new("/nonexistent");
15595        apply_child_env(&mut candidate, &spec(Vec::new()));
15596        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15597        assert_eq!(
15598            role(&candidate),
15599            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15600        );
15601    }
15602
15603    /// Daemon-private capture retention keys never reach the child.
15604    ///
15605    /// cortexkit-log exposes retention as a Rust struct with no environment
15606    /// names, so these entries are supervisor metadata. Passing them through
15607    /// would invent a public child-process contract by accident.
15608    #[test]
15609    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15610        let mut command = Command::new("/nonexistent");
15611        apply_child_env(
15612            &mut command,
15613            &spec(vec![
15614                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15615                ("KEPT".to_string(), "yes".to_string()),
15616            ]),
15617        );
15618        let keys: Vec<String> = command
15619            .as_std()
15620            .get_envs()
15621            .filter(|(_, value)| value.is_some())
15622            .map(|(key, _)| key.to_string_lossy().into_owned())
15623            .collect();
15624        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15625        assert!(
15626            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15627            "daemon-private capture key leaked to the child: {keys:?}"
15628        );
15629    }
15630}
15631
15632#[cfg(test)]
15633mod jitter_tests {
15634    use super::jittered_health_delay;
15635    use std::{collections::HashSet, time::Duration};
15636
15637    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15638    /// that actually occur rather than invented ones.
15639    ///
15640    /// This is a SAMPLE, not a registry: the property under test is that distinct
15641    /// ids disperse, which holds for any set of distinct strings. Several entries
15642    /// are already historical (modules get renamed), and that costs nothing here --
15643    /// but it means a reader must not mistake this for the live module set, and a
15644    /// rename sweep will match it without there being anything to change.
15645    const FLEET: [&str; 14] = [
15646        "aft",
15647        "alfonso-core",
15648        "magic-context",
15649        "broca",
15650        "thalamus",
15651        "quota",
15652        "engram",
15653        "plexus",
15654        "cerebellum",
15655        "astrocyte",
15656        "synapse",
15657        "subc-mcp",
15658        "cortexkit-credentials",
15659        "subc-federation",
15660    ];
15661
15662    /// Probes must not converge after a fleet-wide restart.
15663    ///
15664    /// This is the property the jitter exists for: every module reconnects at
15665    /// once, and without dispersal all fourteen would then probe on the same
15666    /// tick forever. Nothing failed visibly when this went untested -- a
15667    /// convergent fleet still probes correctly, just in a burst, so the symptom
15668    /// is a periodic load spike that looks like whatever else is running.
15669    #[test]
15670    fn probe_delays_disperse_across_the_fleet() {
15671        let cadence = Duration::from_secs(30);
15672        let delays: HashSet<Duration> = FLEET
15673            .iter()
15674            .map(|id| jittered_health_delay(id, 0, cadence))
15675            .collect();
15676        assert_eq!(
15677            delays.len(),
15678            FLEET.len(),
15679            "every supervised module must land on its own probe offset"
15680        );
15681    }
15682
15683    /// The offset may only ever DELAY a probe, never bring it forward.
15684    ///
15685    /// A delay below the cadence would probe a module more often than
15686    /// configured, which is the opposite of what an operator asked for and
15687    /// would tighten the failure budget without anyone changing it.
15688    #[test]
15689    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15690        let cadence = Duration::from_secs(30);
15691        let span = cadence / 10;
15692        for id in FLEET {
15693            for probe_index in 0..8 {
15694                let delay = jittered_health_delay(id, probe_index, cadence);
15695                assert!(
15696                    delay >= cadence,
15697                    "{id}#{probe_index}: jitter must not shorten the cadence"
15698                );
15699                assert!(
15700                    delay < cadence + span,
15701                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15702                );
15703            }
15704        }
15705    }
15706
15707    /// A module keeps its offset across daemon restarts.
15708    ///
15709    /// The delay is derived rather than randomised precisely so a restart does
15710    /// not re-roll every module into a fresh chance of collision. A random
15711    /// source would satisfy the dispersal test above and quietly lose this.
15712    #[test]
15713    fn a_module_offset_is_stable_across_restarts() {
15714        let cadence = Duration::from_secs(30);
15715        for id in FLEET {
15716            assert_eq!(
15717                jittered_health_delay(id, 0, cadence),
15718                jittered_health_delay(id, 0, cadence),
15719                "{id}: the same module and probe index must produce the same offset"
15720            );
15721        }
15722    }
15723
15724    /// A zero cadence disables probing rather than producing a busy loop.
15725    #[test]
15726    fn zero_cadence_yields_zero_delay() {
15727        assert_eq!(
15728            jittered_health_delay("aft", 0, Duration::ZERO),
15729            Duration::ZERO
15730        );
15731    }
15732}
15733
15734#[cfg(all(test, target_os = "linux"))]
15735mod cgroup_placement_tests {
15736    use super::{
15737        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15738        SuperviseError, SupervisedChild,
15739    };
15740    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15741    use cortexkit_test_support::ScratchDir;
15742    use std::{
15743        fs, io,
15744        path::{Path, PathBuf},
15745        sync::{Arc, Mutex},
15746    };
15747    use tokio::process::Command;
15748
15749    #[tokio::test]
15750    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15751        use super::*;
15752        let dir = ScratchDir::new("unique-spawn-cgroups");
15753        let root = PathBuf::from(format!(
15754            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15755            std::process::id(),
15756            unix_ms_now()
15757        ));
15758        if let Err(error) = fs::create_dir(&root) {
15759            assert!(
15760                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15761                "required cgroup test cannot execute: {error}"
15762            );
15763            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15764            return;
15765        }
15766        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15767        let group_count = || {
15768            fs::read_dir(root.join("subc-modules"))
15769                .unwrap()
15770                .map(|entry| entry.unwrap().file_type().unwrap())
15771                .filter(|kind| kind.is_dir())
15772                .count()
15773        };
15774        let supervisor =
15775            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15776                .with_cgroup_placement(Some(placement.clone()));
15777        let runtime = supervisor.runtime_config();
15778        let mut spec = ModuleSpec {
15779            module_id: "unique-spawn".into(),
15780            program: PathBuf::from("/bin/sleep"),
15781            args: vec!["60".into()],
15782            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15783                .into_iter()
15784                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15785                .collect(),
15786            reserved: false,
15787            reserved_prefixes: vec![],
15788            protocol: ModuleProtocol::None,
15789            overlap: Default::default(),
15790        };
15791        let spawn = |spec: &ModuleSpec| {
15792            spawn_child(
15793                spec,
15794                None,
15795                None,
15796                &runtime.stderr_ring,
15797                None,
15798                &runtime.child_roster,
15799                Some(&placement),
15800            )
15801            .unwrap()
15802        };
15803        let mut live = spawn(&spec);
15804        for _ in 0..3 {
15805            // A new process can enter the old slot while retirement is pending.
15806            let next = spawn(&spec);
15807            assert_ne!(live.module_id, next.module_id);
15808            live.start_kill().unwrap();
15809            live.wait().await.unwrap();
15810            live = next;
15811            assert!(
15812                live.child.try_wait().unwrap().is_none(),
15813                "retiring the old slot must not kill the replacement"
15814            );
15815            assert_eq!(
15816                group_count(),
15817                1,
15818                "only the live spawn's cgroup should remain"
15819            );
15820        }
15821        supervisor.begin_daemon_shutdown();
15822        let reap = tokio::spawn(async move {
15823            live.wait().await.unwrap();
15824        });
15825        supervisor
15826            .end_children_for_daemon_shutdown(false, std::future::pending())
15827            .await;
15828        reap.await.unwrap();
15829        assert_eq!(group_count(), 0);
15830        // A normal exit uses the same tree-cleanup path as a killed spawn.
15831        spec.program = PathBuf::from("/bin/true");
15832        spec.args.clear();
15833        let fresh_roster = ChildRoster::default();
15834        let mut short = spawn_child(
15835            &spec,
15836            None,
15837            None,
15838            &runtime.stderr_ring,
15839            None,
15840            &fresh_roster,
15841            Some(&placement),
15842        )
15843        .unwrap();
15844        short.wait().await.unwrap();
15845        assert_eq!(group_count(), 0);
15846        spec.module_id = "_".repeat(255);
15847        let mut long_id = spawn_child(
15848            &spec,
15849            None,
15850            None,
15851            &runtime.stderr_ring,
15852            None,
15853            &fresh_roster,
15854            Some(&placement),
15855        )
15856        .unwrap();
15857        long_id.wait().await.unwrap();
15858        assert_eq!(
15859            group_count(),
15860            0,
15861            "valid long module IDs must not exceed cgroup NAME_MAX"
15862        );
15863        fs::remove_dir(root.join("subc-modules")).unwrap();
15864        fs::remove_dir(root).unwrap();
15865        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15866    }
15867
15868    #[test]
15869    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15870        let path = Path::new("/definitely-missing-subc-cgroup");
15871        let mut command = Command::new("true");
15872        let error = apply_cgroup_placement(
15873            &mut command,
15874            &ModuleSpec {
15875                module_id: "broken-cgroup".to_string(),
15876                program: PathBuf::from("true"),
15877                args: Vec::new(),
15878                env: Vec::new(),
15879                reserved: false,
15880                reserved_prefixes: Vec::new(),
15881                protocol: ModuleProtocol::Subc,
15882                overlap: Default::default(),
15883            },
15884            path,
15885        )
15886        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15887        let reason = error.to_string();
15888
15889        assert!(
15890            matches!(error, SuperviseError::Cgroup { .. }),
15891            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15892        );
15893        assert!(
15894            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15895            "parent cgroup open failure must name cgroup.procs: {reason}"
15896        );
15897    }
15898
15899    #[tokio::test]
15900    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15901        let root = ScratchDir::new("supervisor-reap-cgroup");
15902        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15903        let placement = subc_cgroup::prepare_at(&root)
15904            .expect("prepare scratch cgroup root")
15905            .expect("scratch root has a cgroup.procs marker");
15906        let module_id = "reaped-module";
15907        let module = placement
15908            .module_path(module_id)
15909            .expect("create scratch module cgroup");
15910        let child = Command::new("true")
15911            .env("XDG_DATA_HOME", root.path())
15912            .env("XDG_RUNTIME_DIR", root.path())
15913            .env("XDG_CONFIG_HOME", root.path())
15914            .spawn()
15915            .expect("spawn short-lived child");
15916        let pid = child.id().expect("spawned child has pid");
15917        let mut child = SupervisedChild {
15918            child,
15919            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15920            protocol: ModuleProtocol::Subc,
15921            module_id: module_id.to_string(),
15922            cgroup_placement: Some(placement),
15923            stdout_pump: None,
15924            stderr_pump: None,
15925            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15926            spawned_at: tokio::time::Instant::now(),
15927            spawned_at_ms: 0,
15928            spawned_from: PathBuf::from("true"),
15929            launch_spec_digest: LaunchSpecDigest([0; 32]),
15930            spawned_file_identity: None,
15931            process_start_time: None,
15932            process_identity: None,
15933            pid,
15934            roster_guard: None,
15935            #[cfg(target_os = "macos")]
15936            privacy_exec: None,
15937            spawn_failure: None,
15938        };
15939
15940        child.wait().await.expect("reap short-lived child");
15941
15942        assert!(
15943            !module.exists(),
15944            "reaping the supervised child must remove its empty cgroup"
15945        );
15946    }
15947
15948    #[test]
15949    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15950        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15951        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15952        let placement = subc_cgroup::prepare_at(&root)
15953            .expect("prepare scratch cgroup root")
15954            .expect("scratch root has a cgroup.procs marker");
15955        let module = placement
15956            .module_path("surviving-module")
15957            .expect("create scratch module cgroup");
15958        fs::write(module.join("surviving-process"), b"still present")
15959            .expect("make scratch cgroup non-empty");
15960        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15961
15962        remove_module_cgroup(&placement, "surviving-module");
15963
15964        let logs = crate::router::test_log::captured_logs(&logs);
15965        assert!(
15966            module.exists(),
15967            "failed removal must leave the cgroup intact"
15968        );
15969        assert!(
15970            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15971                && logs.contains("surviving-module"),
15972            "best-effort removal must report the failure without returning it: {logs}"
15973        );
15974    }
15975
15976    #[test]
15977    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15978        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15979        let reason = SuperviseError::Spawn {
15980            program: PathBuf::from("/bin/true"),
15981            source: io::Error::from_raw_os_error(13),
15982            cgroup_path: Some(cgroup_path.clone()),
15983        }
15984        .to_string();
15985
15986        assert!(
15987            reason.contains(&cgroup_path.display().to_string()),
15988            "a pre_exec spawn failure must name the cgroup path: {reason}"
15989        );
15990    }
15991}
15992
15993#[cfg(test)]
15994mod spawn_subscriber_lag_tests {
15995    use super::*;
15996
15997    /// A subscriber whose connection stops draining is dropped once its frame
15998    /// channel fills. The client must learn that from a terminal Error frame
15999    /// after the frames already queued for it, not from a stream that simply
16000    /// goes quiet.
16001    #[tokio::test]
16002    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16003        let feed = SpawnEventFeed::default();
16004        feed.configure_incarnation("lag-incarnation".to_string());
16005        // A one-slot connection queue that nobody reads until the emits are
16006        // done: the forwarder parks on it and the subscriber channel fills.
16007        let (tx, mut rx) = mpsc::channel(1);
16008        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16009            .expect("subscribe");
16010        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16011        for index in 0..emitted {
16012            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16013            // Let the forwarder take what it can so the fill point is the
16014            // subscriber channel, not a scheduling accident.
16015            tokio::task::yield_now().await;
16016        }
16017        assert_eq!(
16018            feed.subscriber_count(),
16019            0,
16020            "the lagged subscriber must be removed"
16021        );
16022
16023        let mut data = Vec::new();
16024        let mut last = None;
16025        loop {
16026            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16027                .await
16028                .expect("the forwarder must finish once the subscriber is dropped");
16029            let Some(outbound) = next else { break };
16030            let frame = outbound.frame;
16031            if frame.header.ty == FrameType::StreamData {
16032                assert!(last.is_none(), "no data may follow the terminal frame");
16033                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16034                data.push(event.cursor.seq);
16035            } else {
16036                assert!(last.is_none(), "exactly one terminal frame");
16037                last = Some(frame);
16038            }
16039        }
16040        assert!(!data.is_empty(), "queued frames drain before the terminal");
16041        for pair in data.windows(2) {
16042            assert_eq!(
16043                pair[1],
16044                pair[0] + 1,
16045                "queued frames arrive dense and in order"
16046            );
16047        }
16048        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16049        assert_eq!(terminal.header.ty, FrameType::Error);
16050        assert_eq!(terminal.header.corr, 7);
16051        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16052        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16053        let detail = body.detail.expect("lagged error carries detail");
16054        assert_eq!(
16055            detail["first_undelivered_cursor"]["seq"],
16056            data.last().unwrap() + 1,
16057            "the named cursor is the first event the subscriber did not receive"
16058        );
16059        assert_eq!(
16060            detail["first_undelivered_cursor"]["daemon_incarnation"],
16061            "lag-incarnation"
16062        );
16063    }
16064}
16065
16066#[cfg(test)]
16067mod terminal_history_read_concurrency_tests {
16068    use super::*;
16069    use crate::terminal_journal::read_pause;
16070    use cortexkit_test_support::ScratchDir;
16071    use std::sync::mpsc as std_mpsc;
16072
16073    fn journaled_ring(
16074        journal: &Arc<crate::terminal_journal::TerminalJournal>,
16075    ) -> Arc<Mutex<TerminalRing>> {
16076        Arc::new(Mutex::new(
16077            TerminalRing::new(TerminalRingConfig::default(), 1)
16078                .with_journal(Some(Arc::clone(journal))),
16079        ))
16080    }
16081
16082    fn crash(at_ms: u64) -> ExitReport {
16083        ExitReport {
16084            kind: ExitKind::Crash,
16085            code: Some(1),
16086            signal: None,
16087            at_ms,
16088        }
16089    }
16090
16091    /// Record an exit on another thread and report whether it finished within
16092    /// `bound`. The recorder thread is left running if it did not.
16093    fn record_within(
16094        module_id: &'static str,
16095        ring: &Arc<Mutex<TerminalRing>>,
16096        at_ms: u64,
16097        bound: Duration,
16098    ) -> bool {
16099        let ring = Arc::clone(ring);
16100        let (done, done_rx) = std_mpsc::channel();
16101        std::thread::spawn(move || {
16102            record_terminal(
16103                module_id,
16104                &ring,
16105                &SpawnEventFeed::default(),
16106                &crash(at_ms),
16107                TerminalDisposition::Restarting,
16108            );
16109            let _ = done.send(());
16110        });
16111        done_rx.recv_timeout(bound).is_ok()
16112    }
16113
16114    /// A history read in progress must not hold the journal writer (which every
16115    /// module's exit recording needs) or the module's own ring. Exits recorded
16116    /// while the read is paused complete promptly; the paused read answers as of
16117    /// the moment it started, and the next read has each exit exactly once.
16118    #[test]
16119    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16120        let dir = ScratchDir::new("terminal-history-concurrent-read");
16121        let path = dir.join("terminals.jsonl");
16122        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16123            path.clone(),
16124            "daemon".into(),
16125        ));
16126        let reader_ring = journaled_ring(&journal);
16127        let other_ring = journaled_ring(&journal);
16128        assert!(record_within(
16129            "reader-module",
16130            &reader_ring,
16131            10,
16132            Duration::from_secs(5)
16133        ));
16134
16135        let (started, release) = read_pause::install(&path);
16136        let reading = {
16137            let ring = Arc::clone(&reader_ring);
16138            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16139        };
16140        started
16141            .recv_timeout(Duration::from_secs(5))
16142            .expect("the history read reached its pause");
16143
16144        let bound = Duration::from_secs(1);
16145        assert!(
16146            record_within("other-module", &other_ring, 20, bound),
16147            "another module's exit waited on a history read (journal writer held)"
16148        );
16149        assert!(
16150            record_within("reader-module", &reader_ring, 30, bound),
16151            "the read module's own exit waited on its history read (ring held)"
16152        );
16153
16154        drop(release);
16155        let paused = reading.join().unwrap();
16156        assert_eq!(
16157            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16158            vec![10],
16159            "an exit recorded after the read began lands in neither half of it"
16160        );
16161        assert_eq!(paused.journal_skipped_lines, 0);
16162        assert_eq!(paused.journal_read_errors, 0);
16163
16164        let after = durable_terminal_history_of(&reader_ring, "reader-module");
16165        assert_eq!(
16166            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16167            vec![10, 30],
16168            "the next read merges ring and journal with no duplicate"
16169        );
16170        assert_eq!(after.journal_skipped_lines, 0);
16171    }
16172}
16173
16174/// What a restart does with the exited process's output readers. These drive
16175/// the same settle functions the supervisor calls, with a reader the test
16176/// holds, so a reader that has not been scheduled by the bound is a controlled
16177/// input rather than something only a loaded machine produces.
16178#[cfg(test)]
16179mod stderr_settle_tests {
16180    use std::{
16181        future::Future,
16182        io,
16183        pin::Pin,
16184        sync::{Arc, Mutex},
16185        task::{Context, Poll},
16186        time::Duration,
16187    };
16188
16189    use tokio::{
16190        io::{AsyncRead, ReadBuf},
16191        sync::oneshot,
16192        time::Instant,
16193    };
16194
16195    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16196    use crate::stderr_tail::{
16197        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16198        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16199    };
16200
16201    const BOUND: Duration = Duration::from_millis(250);
16202
16203    /// Yields `before`, then stays pending until the gate is released, then
16204    /// yields `after` and reaches EOF. The bytes after the gate were written
16205    /// by a process that has already exited; only the reader is behind.
16206    struct HeldReader {
16207        before: Option<Vec<u8>>,
16208        gate: Option<oneshot::Receiver<()>>,
16209        after: io::Cursor<Vec<u8>>,
16210    }
16211
16212    impl AsyncRead for HeldReader {
16213        fn poll_read(
16214            mut self: Pin<&mut Self>,
16215            cx: &mut Context<'_>,
16216            buf: &mut ReadBuf<'_>,
16217        ) -> Poll<io::Result<()>> {
16218            if let Some(bytes) = self.before.take() {
16219                buf.put_slice(&bytes);
16220                return Poll::Ready(Ok(()));
16221            }
16222            if let Some(gate) = self.gate.as_mut() {
16223                match Pin::new(gate).poll(cx) {
16224                    Poll::Pending => return Poll::Pending,
16225                    Poll::Ready(_) => self.gate = None,
16226                }
16227            }
16228            Pin::new(&mut self.after).poll_read(cx, buf)
16229        }
16230    }
16231
16232    struct DiscardSink;
16233
16234    impl OutputSink for DiscardSink {
16235        fn write_line(&mut self, _line: &[u8]) {}
16236    }
16237
16238    fn line(text: &str) -> TailEntry {
16239        TailEntry::Line {
16240            text: text.to_string(),
16241            truncated: false,
16242            at_ms: None,
16243        }
16244    }
16245
16246    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16247        ring.lock().unwrap()
16248    }
16249
16250    /// Start a reader for a new process generation that delivers `before`
16251    /// immediately and `after` only once the returned sender fires (or is
16252    /// dropped).
16253    fn held_pump(
16254        ring: &Arc<Mutex<StderrRing>>,
16255        before: &str,
16256        after: &str,
16257    ) -> (StderrPump, oneshot::Sender<()>) {
16258        let generation = lock(ring).begin_process();
16259        let (release, gate) = oneshot::channel();
16260        let reader = HeldReader {
16261            before: Some(before.as_bytes().to_vec()),
16262            gate: Some(gate),
16263            after: io::Cursor::new(after.as_bytes().to_vec()),
16264        };
16265        let task = tokio::spawn(pump_stderr_to(
16266            reader,
16267            Arc::clone(ring),
16268            generation,
16269            DiscardSink,
16270        ));
16271        (StderrPump { task, generation }, release)
16272    }
16273
16274    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16275        for _ in 0..1000 {
16276            if done(&lock(ring)) {
16277                return;
16278            }
16279            tokio::time::sleep(Duration::from_millis(1)).await;
16280        }
16281        panic!(
16282            "ring never reached the expected state: {:?}",
16283            lock(ring).snapshot(None, None)
16284        );
16285    }
16286
16287    #[tokio::test(start_paused = true)]
16288    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16289        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16290        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16291
16292        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16293        let before_release = lock(&ring).snapshot(None, None);
16294        assert!(
16295            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16296            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16297        );
16298
16299        // The restart: the next process starts and writes before the old
16300        // reader catches up.
16301        let next = lock(&ring).begin_process();
16302        lock(&ring).push_line_from(next, "next process booting");
16303        release.send(()).unwrap();
16304        wait_until(&ring, |ring| {
16305            ring.snapshot(None, None).capture == CaptureState::Captured
16306        })
16307        .await;
16308
16309        assert_eq!(
16310            untimed(lock(&ring).snapshot(None, None).entries),
16311            vec![
16312                line("booting"),
16313                line("config error: missing storage"),
16314                TailEntry::ProcessStart,
16315                line("next process booting"),
16316            ],
16317            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16318        );
16319    }
16320
16321    #[tokio::test(start_paused = true)]
16322    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16323    ) {
16324        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16325        // `_held` is never fired: a descendant keeps the pipe open for the
16326        // whole test.
16327        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16328
16329        let started = Instant::now();
16330        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16331        assert_eq!(
16332            started.elapsed(),
16333            BOUND,
16334            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16335        );
16336
16337        let next = lock(&ring).begin_process();
16338        lock(&ring).push_line_from(next, "next process booting");
16339        tokio::time::sleep(Duration::from_secs(60)).await;
16340
16341        let snapshot = lock(&ring).snapshot(None, None);
16342        match &snapshot.capture {
16343            CaptureState::Incomplete { reason } => assert!(
16344                reason.contains("had not reached EOF") && reason.contains("250ms"),
16345                "the reason must say what is missing and after how long: {reason}"
16346            ),
16347            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16348        }
16349        assert_eq!(
16350            untimed(snapshot.entries),
16351            vec![
16352                line("parent exiting"),
16353                TailEntry::ProcessStart,
16354                line("next process booting"),
16355            ]
16356        );
16357    }
16358
16359    #[tokio::test(start_paused = true)]
16360    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16361        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16362        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16363        release.send(()).unwrap();
16364
16365        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16366
16367        let snapshot = lock(&ring).snapshot(None, None);
16368        assert_eq!(snapshot.capture, CaptureState::Captured);
16369        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16370    }
16371
16372    #[tokio::test(start_paused = true)]
16373    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16374        const BURST: usize = 150;
16375        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16376        let path = capture.join("two-pipe.stderr.log");
16377        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16378        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16379        let generation = lock(&ring).begin_process();
16380        let stdout = (0..BURST)
16381            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16382            .collect::<String>();
16383        let stderr = (0..BURST)
16384            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16385            .collect::<String>();
16386        // Model a child that has written both bursts and exited. Hold the stdout
16387        // reader after one line and a partial prefix, as if pipe readiness or
16388        // capture-file writes had delayed it. The final line also needs EOF to
16389        // flush: neither complete nor partial buffered output may be discarded.
16390        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16391        let (release, gate) = oneshot::channel();
16392        let reader = HeldReader {
16393            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16394            gate: Some(gate),
16395            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16396        };
16397        let (finished, completion) = oneshot::channel();
16398        let stdout_sink = sink.clone();
16399        let pump = tokio::spawn(async move {
16400            pump_stdout_to(reader, stdout_sink).await;
16401            finished.send(()).unwrap();
16402        });
16403        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16404
16405        let started = Instant::now();
16406        settle_stdout_pump("two-pipe", pump, BOUND).await;
16407        assert_eq!(
16408            started.elapsed(),
16409            BOUND,
16410            "a late reader must not hold up restart"
16411        );
16412        assert_eq!(
16413            std::fs::read_to_string(&path).unwrap().lines().count(),
16414            BURST + 1
16415        );
16416
16417        release
16418            .send(())
16419            .expect("stdout reader must survive the exit drain bound");
16420        tokio::time::timeout(Duration::from_secs(1), completion)
16421            .await
16422            .expect("released stdout reader must reach EOF")
16423            .expect("stdout capture must finish at EOF, not be cancelled");
16424        let contents = std::fs::read_to_string(&path).unwrap();
16425        let mut lines = contents
16426            .lines()
16427            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16428            .collect::<Vec<_>>();
16429        let mut expected = (0..BURST)
16430            .flat_map(|i| {
16431                [
16432                    format!("out-{i:04}-{}", "o".repeat(64)),
16433                    format!("err-{i:04}-{}", "e".repeat(64)),
16434                ]
16435            })
16436            .collect::<Vec<_>>();
16437        lines.sort_unstable();
16438        expected.sort_unstable();
16439        assert_eq!(
16440            lines, expected,
16441            "both pipes must retain every complete line exactly once"
16442        );
16443    }
16444}
16445
16446/// Containment of a module's process tree (issue #109).
16447///
16448/// The behaviour these defend against is a module helper surviving its module:
16449/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
16450/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
16451/// compounds it.
16452///
16453/// They run against the SUPERVISOR rather than the job-object crate because the
16454/// claim is about teardown: a crate-level test proves a job can reap a tree, not
16455/// that the daemon's drain path reaches it.
16456///
16457/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
16458/// lane there is a separate containment path with its own tests.
16459#[cfg(all(test, windows))]
16460mod job_containment_tests {
16461    use super::*;
16462    use cortexkit_test_support::ScratchDir;
16463    use std::{
16464        path::{Path, PathBuf},
16465        sync::{Arc, Mutex},
16466        time::{Duration, Instant},
16467    };
16468
16469    /// The stub, expected beside this test executable.
16470    ///
16471    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
16472    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
16473    /// failure then reads as a broken test rather than an unbuilt dependency.
16474    fn stub_path() -> PathBuf {
16475        let mut path = std::env::current_exe().expect("current_exe available in tests");
16476        path.pop();
16477        path.pop();
16478        path.push("fake-aft-stub.exe");
16479        assert!(
16480            path.exists(),
16481            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16482             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16483            path.display()
16484        );
16485        path
16486    }
16487
16488    /// Poll for the grandchild pid the stub records, and parse it.
16489    fn read_grandchild_pid(path: &Path) -> u32 {
16490        let deadline = Instant::now() + Duration::from_secs(10);
16491        loop {
16492            if let Ok(contents) = std::fs::read_to_string(path) {
16493                if let Ok(pid) = contents.trim().parse() {
16494                    return pid;
16495                }
16496            }
16497            assert!(
16498                Instant::now() < deadline,
16499                "the stub never recorded a grandchild pid at {}",
16500                path.display()
16501            );
16502            std::thread::sleep(Duration::from_millis(10));
16503        }
16504    }
16505
16506    /// Everything one fixture run needs, so the two tests below differ in exactly
16507    /// one place: whether the child is contained.
16508    struct Fixture {
16509        _dir: ScratchDir,
16510        module_id: String,
16511        grandchild: u32,
16512        child: Option<SupervisedChild>,
16513        registry: Arc<Registry>,
16514        snapshot: Arc<Mutex<SupervisorSnapshot>>,
16515        terminal_ring: Arc<Mutex<TerminalRing>>,
16516        spawn_events: SpawnEventFeed,
16517    }
16518
16519    fn fixture(label: &str, module_id: &str) -> Fixture {
16520        let dir = ScratchDir::new(label);
16521        let pid_file = dir.join("grandchild.pid");
16522        let supervisor = Supervisor::new_for_test(
16523            Arc::new(Registry::default()),
16524            RestartPolicy::new(3, Duration::ZERO),
16525        );
16526        let runtime = supervisor.runtime_config();
16527        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16528        let spec = ModuleSpec {
16529            module_id: module_id.to_string(),
16530            program: stub_path(),
16531            // Zero args deliberately: a `--subc` argument would make the stub dial
16532            // a daemon that is not there, and the failure would land in the same
16533            // stderr ring this fixture exists to keep quiet.
16534            args: Vec::new(),
16535            env: vec![
16536                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16537                (
16538                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16539                    pid_file.display().to_string(),
16540                ),
16541            ],
16542            reserved: false,
16543            reserved_prefixes: Vec::new(),
16544            protocol: ModuleProtocol::Subc,
16545            overlap: Default::default(),
16546        };
16547        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16548            .expect("spawn the supervised fixture");
16549        let grandchild = read_grandchild_pid(&pid_file);
16550        Fixture {
16551            _dir: dir,
16552            module_id: module_id.to_string(),
16553            grandchild,
16554            child: Some(child),
16555            registry: Arc::new(Registry::default()),
16556            snapshot,
16557            terminal_ring: Arc::clone(&runtime.terminal_ring),
16558            spawn_events: SpawnEventFeed::default(),
16559        }
16560    }
16561
16562    impl Fixture {
16563        /// Drain through the supervisor's own teardown path.
16564        async fn drain(&mut self) {
16565            let child = self
16566                .child
16567                .take()
16568                .expect("the fixture child is still present");
16569            drain_child_to_state(
16570                &self.module_id,
16571                ModuleProtocol::Subc,
16572                // No forwarding table in this fixture, so nothing reaches the
16573                // child over a connection.
16574                StopNotice::NotSent,
16575                &self.registry,
16576                None,
16577                &self.snapshot,
16578                &self.terminal_ring,
16579                &self.spawn_events,
16580                child,
16581                Duration::from_millis(500),
16582                ModuleState::Stopped,
16583                Some(false),
16584                &FailureContext::operator(),
16585            )
16586            .await
16587            .expect("drain the supervised fixture");
16588        }
16589    }
16590
16591    /// Teardown reaps the grandchild, not merely the direct child.
16592    ///
16593    /// This is the assertion the change exists for. Before containment the
16594    /// grandchild survived: it is a separate process, and `start_kill` is
16595    /// `TerminateProcess` scoped to one pid.
16596    #[tokio::test]
16597    async fn teardown_reaps_the_grandchild() {
16598        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16599        let grandchild = fixture.grandchild;
16600
16601        assert!(
16602            subc_jobobject::process_exists(grandchild),
16603            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16604        );
16605
16606        fixture.drain().await;
16607
16608        assert!(
16609            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16610            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16611        );
16612    }
16613
16614    /// The mutation control: with containment withheld, the grandchild survives
16615    /// the same kill.
16616    ///
16617    /// This is the defect reproduction from #109 — a direct-child kill reaches
16618    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16619    /// supervisor because `spawn_and_mark_running` now always contains on
16620    /// Windows, which is the point: there is no longer a path that spawns
16621    /// uncontained, so the control has to construct one.
16622    ///
16623    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16624    /// grandchild ever dies here, that test is passing for a reason unrelated to
16625    /// the job object and the containment claim is unproven.
16626    #[test]
16627    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16628        let dir = ScratchDir::new("teardown-uncontained");
16629        let pid_file = dir.join("grandchild.pid");
16630        let mut child = std::process::Command::new(stub_path())
16631            .env("FAKE_AFT_NEVER_CONNECT", "1")
16632            .env(
16633                "FAKE_AFT_GRANDCHILD_PID_FILE",
16634                pid_file.display().to_string(),
16635            )
16636            .stdin(std::process::Stdio::null())
16637            .stdout(std::process::Stdio::null())
16638            .stderr(std::process::Stdio::null())
16639            .spawn()
16640            .expect("spawn the uncontained fixture");
16641        let grandchild = read_grandchild_pid(&pid_file);
16642
16643        // Exactly what the pre-fix teardown did: kill the direct child.
16644        child.kill().expect("kill the direct child");
16645        let _ = child.wait();
16646
16647        assert!(
16648            subc_jobobject::process_exists(grandchild),
16649            "grandchild {grandchild} died with the direct child, so this control no longer \
16650             distinguishes contained from uncontained teardown and the regression test is \
16651             passing vacuously"
16652        );
16653
16654        // The orphan this control demonstrates is the leak the fix prevents, so
16655        // the control must not leave one behind.
16656        kill_tree(grandchild);
16657    }
16658
16659    /// Crash durability: closing the containment handle reaps the tree with no
16660    /// teardown code running at all.
16661    ///
16662    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16663    /// call anything — and it is why containment is a kernel property of the
16664    /// handle rather than a step in the drain. Discovered by getting the
16665    /// mutation control wrong: clearing `job` to "disable" containment instead
16666    /// killed the tree, which is the guarantee, not a mistake.
16667    #[tokio::test]
16668    async fn dropping_containment_reaps_the_grandchild() {
16669        let mut fixture = fixture("drop-containment", "tree-drop");
16670        let grandchild = fixture.grandchild;
16671
16672        assert!(subc_jobobject::process_exists(grandchild));
16673
16674        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16675        fixture.child.as_mut().expect("child present").job = None;
16676
16677        assert!(
16678            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16679            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16680             crash would leave the tree behind"
16681        );
16682    }
16683
16684    /// Kill a pid and its tree, then confirm it is gone.
16685    fn kill_tree(pid: u32) {
16686        let _ = std::process::Command::new("taskkill.exe")
16687            .args(["/PID", &pid.to_string(), "/T", "/F"])
16688            .stdin(std::process::Stdio::null())
16689            .stdout(std::process::Stdio::null())
16690            .stderr(std::process::Stdio::null())
16691            .status();
16692        assert!(
16693            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16694            "could not clean up grandchild {pid}"
16695        );
16696    }
16697}
16698
16699#[cfg(test)]
16700mod privacy_trampoline_configuration_tests {
16701    #[tokio::test]
16702    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16703    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16704        #[cfg(target_os = "macos")]
16705        {
16706            let supervisor = super::Supervisor::new(
16707                std::sync::Arc::new(crate::Registry::default()),
16708                super::RestartPolicy::default(),
16709            );
16710            let error = supervisor.spawn(spec()).unwrap_err();
16711            assert!(
16712                error
16713                    .to_string()
16714                    .contains("no privacy trampoline configured"),
16715                "{error}"
16716            );
16717        }
16718    }
16719
16720    #[tokio::test]
16721    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16722    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16723        #[cfg(target_os = "macos")]
16724        {
16725            let supervisor = super::Supervisor::new(
16726                std::sync::Arc::new(crate::Registry::default()),
16727                super::RestartPolicy::default(),
16728            )
16729            .with_privacy_trampoline(std::env::current_exe().unwrap());
16730            let error = supervisor.spawn(spec()).unwrap_err();
16731            assert!(
16732                error
16733                    .to_string()
16734                    .contains("binary does not implement the privacy trampoline protocol"),
16735                "{error}"
16736            );
16737        }
16738    }
16739
16740    #[cfg(target_os = "macos")]
16741    fn spec() -> super::ModuleSpec {
16742        super::ModuleSpec {
16743            module_id: "privacy-configuration".into(),
16744            program: "/bin/sleep".into(),
16745            args: vec!["30".into()],
16746            env: vec![],
16747            reserved: false,
16748            reserved_prefixes: vec![],
16749            protocol: subc_control::ModuleProtocol::None,
16750            overlap: super::ModuleOverlap::Exclusive,
16751        }
16752    }
16753}
16754
16755#[cfg(test)]
16756mod privacy_exec_boundary_tests {
16757    #[cfg(target_os = "macos")]
16758    use super::*;
16759    #[cfg(target_os = "macos")]
16760    use std::{
16761        io::{Read, Write},
16762        net::{TcpListener, TcpStream},
16763    };
16764
16765    /// Unit-test-only pause at the actual early image read, not at a later
16766    /// status read. Production supervisors never inspect this environment key.
16767    #[cfg(target_os = "macos")]
16768    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16769        if let Some((_, path)) = spec
16770            .env
16771            .iter()
16772            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16773        {
16774            let mut barrier = TcpStream::connect(path).unwrap();
16775            barrier
16776                .set_read_timeout(Some(Duration::from_secs(30)))
16777                .unwrap();
16778            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16779            let mut release = [0];
16780            barrier.read_exact(&mut release).unwrap();
16781            assert_eq!(&release, b"X");
16782        }
16783    }
16784
16785    #[cfg(target_os = "macos")]
16786    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16787        ModuleSpec {
16788            module_id: "privacy-boundary".into(),
16789            program: program.into(),
16790            args: args.iter().map(|arg| (*arg).into()).collect(),
16791            env: vec![],
16792            reserved: false,
16793            reserved_prefixes: vec![],
16794            protocol: ModuleProtocol::None,
16795            overlap: ModuleOverlap::Exclusive,
16796        }
16797    }
16798
16799    #[cfg(target_os = "macos")]
16800    async fn accept(listener: TcpListener) -> TcpStream {
16801        // Socket readiness, not elapsed time, establishes both pause points.
16802        let listener = tokio::net::TcpListener::from_std({
16803            listener.set_nonblocking(true).unwrap();
16804            listener
16805        })
16806        .unwrap();
16807        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16808            .await
16809            .unwrap()
16810            .unwrap();
16811        let stream = stream.into_std().unwrap();
16812        stream.set_nonblocking(false).unwrap();
16813        stream
16814            .set_read_timeout(Some(Duration::from_secs(30)))
16815            .unwrap();
16816        stream
16817    }
16818
16819    #[tokio::test]
16820    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16821    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16822        #[cfg(target_os = "macos")]
16823        {
16824            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16825            // Loopback sockets also work when the replay adapter's TMPDIR is
16826            // longer than Darwin's Unix-domain socket path limit.
16827            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16828            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16829            let record = root.join("live-children.json");
16830            let supervisor =
16831                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16832                    .with_live_children_record(&record);
16833            let runtime = supervisor.runtime_config();
16834            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16835            let mut spec = spec("/bin/sleep", &["30"]);
16836            spec.env = vec![
16837                (
16838                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16839                    exec_listener.local_addr().unwrap().to_string(),
16840                ),
16841                (
16842                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16843                    sample_listener.local_addr().unwrap().to_string(),
16844                ),
16845            ];
16846            let spawn = tokio::task::spawn_blocking(move || {
16847                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16848            });
16849            let mut sample = accept(sample_listener).await;
16850            let mut pid = [0; 4];
16851            sample.read_exact(&mut pid).unwrap();
16852            let pid = u32::from_ne_bytes(pid);
16853            let mut exec = accept(exec_listener).await;
16854            let mut ready = [0];
16855            exec.read_exact(&mut ready).unwrap();
16856            assert_eq!(&ready, b"R");
16857            // The early read is guaranteed to see a real, non-null trampoline
16858            // image: the fixture has reached its barrier and cannot exec yet.
16859            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16860            assert_eq!(
16861                observe_spawned_image(pid).unwrap().executable,
16862                Some(trampoline)
16863            );
16864            sample.write_all(b"X").unwrap();
16865            let mut child = spawn.await.unwrap();
16866            let early = crate::live_children::read_record(&record).unwrap();
16867            assert_eq!(early.len(), 1);
16868            assert_eq!(early[0].pid, pid);
16869            assert_eq!(
16870                early[0].executable, None,
16871                "unconfirmed trampoline image entered the roster"
16872            );
16873            assert!(child.report_ready.get().is_none());
16874            // The barrier's duration is unrelated to the production five-second
16875            // exec budget. Start the test's confirmation budget upon release.
16876            child.privacy_exec.as_mut().unwrap().deadline =
16877                tokio::time::Instant::now() + Duration::from_secs(30);
16878            exec.write_all(b"X").unwrap();
16879            child.confirm_privacy_exec().await;
16880            assert_eq!(child.spawn_failure, None);
16881            assert!(child.report_ready.get().is_some());
16882            let confirmed = crate::live_children::read_record(&record).unwrap();
16883            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16884            assert_ne!(module, trampoline);
16885            assert_eq!(confirmed[0].executable, Some(module.into()));
16886            child.start_kill().unwrap();
16887            child.wait().await.unwrap();
16888            child.release_roster();
16889        }
16890    }
16891
16892    #[tokio::test]
16893    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16894    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16895        #[cfg(target_os = "macos")]
16896        {
16897            let registry = Arc::new(Registry::default());
16898            let policy = RestartPolicy::new(0, Duration::ZERO);
16899            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16900            let runtime = supervisor.runtime_config();
16901            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16902            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16903            // Drive spawn and confirmation separately instead of starting the
16904            // monitor. WNOWAIT observes a real exit without consuming its status,
16905            // so confirmation's first try_wait must take the already-exited arm.
16906            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16907            let pid = child.pid;
16908            tokio::task::spawn_blocking(move || {
16909                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16910            })
16911            .await
16912            .unwrap()
16913            .unwrap();
16914            child.privacy_exec.as_mut().unwrap().deadline =
16915                tokio::time::Instant::now() + Duration::from_secs(30);
16916            let status = child.wait().await.unwrap();
16917            assert_eq!(status.code(), Some(121));
16918            assert!(child.privacy_exec.is_none());
16919            assert!(
16920                child.report_ready.get().is_none(),
16921                "an exited module must not publish a live pid"
16922            );
16923            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16924            on_child_exit(
16925                &spec,
16926                policy,
16927                &registry,
16928                &snapshot,
16929                &runtime.terminal_ring,
16930                &runtime.spawn_events,
16931                &runtime.child_roster,
16932                report,
16933                true,
16934            )
16935            .await;
16936            let state = lock_snapshot(&snapshot).unwrap();
16937            assert_eq!(state.state, ModuleState::Failed);
16938            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16939            assert_eq!(state.reported_pid(), None);
16940            drop(state);
16941            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16942            assert_eq!(history.entries.len(), 1);
16943            let terminal = &history.entries[0];
16944            assert_eq!(terminal.exit_code, Some(121));
16945            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16946            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16947            assert_eq!(
16948                terminal.disposition_detail.as_deref(),
16949                Some(policy.budget_exhausted_detail().as_str()),
16950                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16951            );
16952            assert_eq!(child.spawn_failure, None);
16953            child.release_roster();
16954        }
16955    }
16956}
16957
16958/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16959/// descriptor 3, without an environment copy. The shell records the nonce
16960/// and its environment after exec so these tests observe the real handover.
16961#[cfg(all(test, unix))]
16962mod launch_nonce_descriptor_tests {
16963    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16964    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16965    use cortexkit_test_support::ScratchDir;
16966    use std::{
16967        path::PathBuf,
16968        sync::{Arc, Mutex},
16969        time::{Duration, Instant},
16970    };
16971
16972    async fn probe(role: super::SpawnRole) {
16973        let scratch = ScratchDir::new("launch-nonce-descriptor");
16974        let fd_copy = scratch.join("from-descriptor");
16975        let env_copy = scratch.join("environment");
16976        let script = format!(
16977            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16978            fd = fd_copy.display(), env = env_copy.display(),
16979        );
16980        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16981        let spec = ModuleSpec {
16982            module_id: "nonce-descriptor-probe".to_string(),
16983            program: PathBuf::from("/bin/sh"),
16984            args: vec!["-c".to_string(), script],
16985            env: vec![
16986                xdg("XDG_DATA_HOME"),
16987                xdg("XDG_RUNTIME_DIR"),
16988                xdg("XDG_CONFIG_HOME"),
16989                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16990            ],
16991            reserved: true,
16992            reserved_prefixes: Vec::new(),
16993            protocol: ModuleProtocol::Subc,
16994            overlap: Default::default(),
16995        };
16996        let handle = SupervisorHandle::new();
16997        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16998        let roster = ChildRoster::default();
16999        #[cfg(target_os = "macos")]
17000        {
17001            let path = super::test_privacy_trampoline();
17002            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17003        }
17004        let child = super::spawn_child_in_slot(
17005            &spec,
17006            None,
17007            Some(&handle),
17008            &ring,
17009            None,
17010            &roster,
17011            #[cfg(target_os = "linux")]
17012            None,
17013            role,
17014            matches!(role, super::SpawnRole::SwapCandidate),
17015        )
17016        .expect("spawn probe");
17017        let deadline = Instant::now() + Duration::from_secs(10);
17018        while !(fd_copy.exists() && env_copy.exists()) {
17019            assert!(Instant::now() < deadline, "probe never wrote its copies");
17020            tokio::time::sleep(Duration::from_millis(20)).await;
17021        }
17022        let nonce = std::fs::read_to_string(fd_copy).unwrap();
17023        assert!(!nonce.is_empty());
17024        let environment = std::fs::read_to_string(env_copy).unwrap();
17025        assert!(environment
17026            .lines()
17027            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17028        let copy = environment
17029            .lines()
17030            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17031        assert_eq!(
17032            copy, None,
17033            "Unix children must never receive the environment nonce"
17034        );
17035        if matches!(role, super::SpawnRole::Plain) {
17036            assert_eq!(
17037                handle.spawn_nonce(&spec.module_id).as_deref(),
17038                Some(nonce.as_str())
17039            );
17040        }
17041        drop(child);
17042    }
17043
17044    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17045    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17046        probe(super::SpawnRole::Plain).await;
17047    }
17048
17049    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17050    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17051        probe(super::SpawnRole::SwapCandidate).await;
17052    }
17053}
17054
17055#[cfg(all(test, target_os = "linux"))]
17056mod cgroup_containment_tests {
17057    use super::*;
17058    use cortexkit_test_support::ScratchDir;
17059
17060    fn running(pid: u32) -> bool {
17061        // An orphan can remain a zombie until the container init reaps it.
17062        std::fs::read_to_string(format!("/proc/{pid}/stat"))
17063            .ok()
17064            .and_then(|stat| {
17065                stat.rsplit_once(") ")
17066                    .map(|(_, rest)| rest.starts_with('Z'))
17067            })
17068            .is_some_and(|zombie| !zombie)
17069    }
17070
17071    #[tokio::test]
17072    async fn linux_teardown_reaps_the_grandchild() {
17073        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17074    }
17075
17076    #[tokio::test]
17077    async fn linux_shutdown_straggler_reaps_the_grandchild() {
17078        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17079    }
17080
17081    async fn teardown_tree(test_name: &str, shutdown: bool) {
17082        let dir = ScratchDir::new(test_name);
17083        let root = PathBuf::from(format!(
17084            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17085            std::process::id(),
17086            unix_ms_now()
17087        ));
17088        if let Err(error) = std::fs::create_dir(&root) {
17089            assert!(
17090                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17091                "required cgroup test cannot execute: {error}"
17092            );
17093            eprintln!(
17094                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17095                root.display()
17096            );
17097            return;
17098        }
17099        let placement = subc_cgroup::prepare_at(&root)
17100            .expect("prepare isolated kernel cgroup")
17101            .expect("isolated cgroup is delegated");
17102        let module_id = "tree-teardown";
17103        let module = placement
17104            .module_path(module_id)
17105            .expect("create isolated module cgroup");
17106        if !module.join("cgroup.kill").exists() {
17107            std::fs::remove_dir(&module).unwrap();
17108            std::fs::remove_dir(root.join("subc-modules")).unwrap();
17109            std::fs::remove_dir(&root).unwrap();
17110            assert!(
17111                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17112                "required cgroup.kill interface unavailable"
17113            );
17114            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17115            return;
17116        }
17117        let supervisor = Supervisor::new_for_test(
17118            Arc::new(Registry::default()),
17119            RestartPolicy::new(3, Duration::ZERO),
17120        )
17121        .with_cgroup_placement(Some(placement));
17122        let mut runtime = supervisor.runtime_config();
17123        runtime.child_roster = runtime
17124            .child_roster
17125            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17126        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17127        let pid_file = dir.join("grandchild.pid");
17128        let spec = ModuleSpec {
17129            module_id: module_id.to_string(),
17130            program: PathBuf::from("/bin/sh"),
17131            args: vec![
17132                "-c".into(),
17133                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17134                "fixture".into(),
17135                pid_file.display().to_string(),
17136            ],
17137            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17138                .into_iter()
17139                .map(|key| (key.to_string(), dir.display().to_string()))
17140                .collect(),
17141            reserved: false,
17142            reserved_prefixes: Vec::new(),
17143            protocol: ModuleProtocol::None,
17144            overlap: Default::default(),
17145        };
17146        let child =
17147            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17148        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17149        let grandchild: u32 = loop {
17150            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17151                if let Ok(pid) = contents.trim().parse() {
17152                    break pid;
17153                }
17154            }
17155            assert!(
17156                tokio::time::Instant::now() < deadline,
17157                "grandchild pid was not recorded"
17158            );
17159            tokio::time::sleep(Duration::from_millis(10)).await;
17160        };
17161        assert!(
17162            running(grandchild),
17163            "grandchild must be alive before teardown"
17164        );
17165        if shutdown {
17166            let mut child = child;
17167            crate::child_roster::end_children_for_daemon_shutdown(
17168                &runtime.child_roster,
17169                false,
17170                std::future::pending(),
17171            )
17172            .await;
17173            child.wait().await.expect("reap shutdown straggler");
17174        } else {
17175            drain_child_to_state(
17176                module_id,
17177                ModuleProtocol::None,
17178                StopNotice::NotSent,
17179                &Registry::default(),
17180                None,
17181                &snapshot,
17182                &runtime.terminal_ring,
17183                &SpawnEventFeed::default(),
17184                child,
17185                Duration::from_millis(100),
17186                ModuleState::Stopped,
17187                Some(false),
17188                &FailureContext::operator(),
17189            )
17190            .await
17191            .expect("real supervisor teardown");
17192        }
17193        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17194        while running(grandchild) && tokio::time::Instant::now() < deadline {
17195            tokio::time::sleep(Duration::from_millis(10)).await;
17196        }
17197        let survived = running(grandchild);
17198        // Kill a surviving grandchild so a failed test does not leave it behind.
17199        if survived {
17200            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17201            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17202            tokio::time::sleep(Duration::from_millis(100)).await;
17203        }
17204        if module.exists() {
17205            std::fs::remove_dir(&module).expect("remove empty module cgroup");
17206        }
17207        std::fs::remove_dir(root.join("subc-modules")).unwrap();
17208        std::fs::remove_dir(&root).unwrap();
17209        assert!(
17210            !survived,
17211            "grandchild {grandchild} outlived module teardown"
17212        );
17213        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17214    }
17215}
17216
17217/// A slow live-children record write on the restart path must not stop the
17218/// daemon from answering the replacement module's authentication handshake.
17219#[cfg(all(test, unix))]
17220mod record_write_stall_tests {
17221    use super::*;
17222    use std::sync::atomic::{AtomicBool, Ordering};
17223    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17224
17225    /// `subc-client-rs` gives the whole handshake this long before the module
17226    /// exits with `Timeout { stage: ServerProof }`.
17227    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17228    /// How long the replacement's admission write is held: longer than the
17229    /// module's deadline, standing in for the delay a busy disk can add
17230    /// across the two syncs in that write.
17231    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17232
17233    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
17234    /// On macOS every module is launched through the privacy trampoline, and
17235    /// a module that is not a `#!` script is admitted only if its running
17236    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
17237    /// the shell soon replaces itself with `/bin/sleep`, which does not match
17238    /// the configured `/bin/sh`. The image is read only after the spawn's
17239    /// live-children record write, and here that write lasted long enough for
17240    /// the shell to have exec'd every time, so each process was refused and
17241    /// killed right after spawn, and the supervisor crash-looped it with
17242    /// backoff until its restart budget ran out. The restart this test
17243    /// measures would then never produce a module at all.
17244    fn sleeping_spec() -> ModuleSpec {
17245        ModuleSpec {
17246            module_id: "record-write-stall".into(),
17247            program: "/bin/sleep".into(),
17248            args: vec!["600".into()],
17249            env: vec![],
17250            reserved: false,
17251            reserved_prefixes: vec![],
17252            protocol: ModuleProtocol::None,
17253            overlap: Default::default(),
17254        }
17255    }
17256
17257    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17258    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17259        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17260        let supervisor =
17261            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17262                .with_live_children_record(home.join("live-children.json"));
17263        let module = supervisor.spawn(sleeping_spec()).unwrap();
17264        // `reported_pid` names a process only once it is confirmed as the
17265        // module (on macOS, once the trampoline's image check has passed), so
17266        // a process the daemon is about to refuse and kill never counts as
17267        // the running module, here or in the final wait below.
17268        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17269            loop {
17270                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17271                    .unwrap()
17272                    .reported_pid()
17273                {
17274                    return pid;
17275                }
17276                tokio::time::sleep(Duration::from_millis(10)).await;
17277            }
17278        })
17279        .await
17280        .expect("the first process was admitted as the module");
17281
17282        // The daemon's listener, served by this runtime's workers like the
17283        // supervisor's own tasks.
17284        let key = vec![0x42; 32];
17285        let daemon_id = [0x24; 16];
17286        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17287        let port = listener.local_addr().unwrap().port();
17288        let server = tokio::spawn(crate::server::serve_listener(
17289            listener,
17290            Arc::new(crate::router::Router::with_default_self_handler()),
17291            crate::server::ServerAuth::with_limits(
17292                key.clone(),
17293                daemon_id,
17294                "test",
17295                crate::server::DEFAULT_AUTH_DEADLINE,
17296                4,
17297            ),
17298        ));
17299
17300        // Hold the write that admits the replacement, once, and say when it
17301        // has started.
17302        let (held_tx, held_rx) = std::sync::mpsc::channel();
17303        let armed = AtomicBool::new(true);
17304        supervisor
17305            .child_roster
17306            .set_record_write_hook_for_test(move |live| {
17307                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17308                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17309                    let _ = held_tx.send(());
17310                    std::thread::sleep(HELD_RECORD_WRITE);
17311                }
17312            });
17313
17314        // The replacement module, as a separate process would be: its own
17315        // thread and runtime, so only the daemon's runtime can be stalled.
17316        let module_process = std::thread::spawn(move || {
17317            held_rx
17318                .recv_timeout(Duration::from_secs(30))
17319                .expect("the restart reached the replacement's record write");
17320            let conn = ConnectionInfo {
17321                schema: SCHEMA_VERSION,
17322                wire_version: None,
17323                endpoints: vec![Endpoint {
17324                    host: "127.0.0.1".to_owned(),
17325                    port,
17326                }],
17327                key,
17328                daemon_id,
17329                pid: std::process::id(),
17330                daemon_ver: "test".to_owned(),
17331            };
17332            tokio::runtime::Builder::new_current_thread()
17333                .enable_all()
17334                .build()
17335                .unwrap()
17336                .block_on(async {
17337                    let started = Instant::now();
17338                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17339                        .await
17340                        .unwrap();
17341                    let result =
17342                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17343                    (result, started.elapsed())
17344                })
17345        });
17346
17347        module.restart(Some(0)).await.unwrap();
17348        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17349            .await
17350            .unwrap();
17351        assert!(
17352            auth.is_ok(),
17353            "the daemon must answer a module's handshake while a restart waits on the \
17354             live-children record write; got {auth:?} after {elapsed:?}"
17355        );
17356
17357        tokio::time::timeout(Duration::from_secs(20), async {
17358            while lock_snapshot(&module.inner.snapshot)
17359                .unwrap()
17360                .reported_pid()
17361                .is_none_or(|pid| pid == old_pid)
17362            {
17363                tokio::time::sleep(Duration::from_millis(10)).await;
17364            }
17365        })
17366        .await
17367        .expect("the restart published its replacement");
17368        module.stop().await.unwrap();
17369        server.abort();
17370    }
17371}