Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    /// Conservative monotonic bound sampled before `command.spawn()`, not when
362    /// identity recording finishes or the spawn generation is published.
363    spawned_at: Instant,
364    spawned_at_ms: u64,
365    spawned_from: PathBuf,
366    launch_spec_digest: LaunchSpecDigest,
367    spawned_file_identity: Option<SpawnedFileIdentity>,
368    process_start_time: Option<u64>,
369    process_identity: Option<ProcessIdentity>,
370    pid: u32,
371    /// This process's entry in the daemon's child roster, released when the
372    /// process is reaped or this handle is dropped.
373    roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377    fn id(&self) -> Option<u32> {
378        Some(self.pid)
379    }
380
381    fn process_identity(&self) -> Option<ProcessIdentity> {
382        self.process_identity
383    }
384
385    async fn wait(&mut self) -> io::Result<ExitStatus> {
386        #[cfg(target_os = "macos")]
387        self.confirm_privacy_exec().await;
388        // The roster entry is NOT released here. A daemon shutdown waits for the
389        // roster to empty and then exits the process, so releasing at the reap
390        // let it exit before the exit handler wrote this child's terminal record
391        // (the stderr drain and snapshot update sit in between), and the
392        // shutdown's own `daemon_shutdown` record was intermittently lost. The
393        // caller releases it after recording the exit (`release_roster`), and
394        // dropping the handle releases it too.
395        let result = self.child.wait().await;
396        #[cfg(target_os = "linux")]
397        if result.is_ok() {
398            if let Some(placement) = self.cgroup_placement.as_ref() {
399                cleanup_reaped_cgroup(placement, &self.module_id).await;
400                // Keep ownership while awaiting kernel population changes: a
401                // drain timeout may cancel this wait and then escalate/reap.
402                self.cgroup_placement = None;
403            }
404        }
405        result
406    }
407
408    #[cfg(target_os = "macos")]
409    async fn confirm_privacy_exec(&mut self) {
410        let Some(pending) = &mut self.privacy_exec else {
411            return;
412        };
413        let result = tokio::time::timeout_at(pending.deadline, async {
414            let mut record = Vec::new();
415            loop {
416                let mut ready = pending.reader.readable().await?;
417                let read = ready.try_io(|reader| {
418                    use std::io::Read;
419                    let mut reader = reader.get_ref();
420                    let mut buffer = [0u8; 256];
421                    reader.read(&mut buffer).map(|count| (count, buffer))
422                });
423                match read {
424                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425                    Ok(Ok((count, buffer))) => {
426                        if record.len() + count > 1024 {
427                            return Err(io::Error::other(
428                                "privacy exec refusal record is too long",
429                            ));
430                        }
431                        record.extend_from_slice(&buffer[..count]);
432                    }
433                    Ok(Err(error)) => return Err(error),
434                    Err(_) => continue,
435                }
436            }
437        })
438        .await;
439        #[cfg(test)]
440        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441            use tokio::io::{AsyncReadExt, AsyncWriteExt};
442            if stream.is_none() {
443                *stream = Some(
444                    tokio::net::TcpStream::connect(address.as_str())
445                        .await
446                        .unwrap(),
447                );
448                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449                stream
450                    .as_mut()
451                    .unwrap()
452                    .write_all(&self.pid.to_ne_bytes())
453                    .await
454                    .unwrap();
455            }
456            let mut reply = [0];
457            stream
458                .as_mut()
459                .unwrap()
460                .read_exact(&mut reply)
461                .await
462                .unwrap();
463            pending.confirmation_pause = None;
464            if &reply == b"F" {
465                Ok(Err(io::Error::other("injected late exec refusal")))
466            } else {
467                assert_eq!(&reply, b"X");
468                result
469            }
470        } else {
471            result
472        };
473        // Keep the reader in self across await: select cancellation must not
474        // discard the handshake or reset its original five-second deadline.
475        let pending = self.privacy_exec.as_ref().expect("pending exec");
476        let cause = match result {
477            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478            Ok(Err(error)) => Some(format!(
479                "privacy identity exec acknowledgement failed: {error}"
480            )),
481            Ok(Ok(record)) if !record.is_empty() => Some(
482                std::str::from_utf8(&record)
483                    .ok()
484                    .and_then(|record| {
485                        record
486                            .trim()
487                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488                    })
489                    .filter(|cause| !cause.is_empty())
490                    .unwrap_or("invalid privacy exec refusal record")
491                    .to_string(),
492            ),
493            Ok(Ok(_)) => match self.child.try_wait() {
494                // Empty EOF is the exec acknowledgement. A real module may exit
495                // immediately, including with a reserved trampoline status; no
496                // image is admitted, and its ordinary exit contract stays intact.
497                Ok(Some(_status)) => None,
498                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499                Ok(None) => {
500                    let image = observe_spawned_image(self.pid);
501                    if let Some(image) = image.filter(|image| {
502                        image.executable.is_some()
503                            && image.executable != pending.trampoline
504                            && (image.executable == pending.expected || pending.script)
505                    }) {
506                        if let Err(error) = self
507                            .roster_guard
508                            .as_ref()
509                            .map(|guard| guard.confirm_executable(image))
510                            .transpose()
511                        {
512                            Some(format!(
513                                "could not durably record confirmed module image: {error}"
514                            ))
515                        } else {
516                            let _ = self.report_ready.set(());
517                            info!(module_id = %pending.module_id, pid = self.pid,
518                                "module spawned with own privacy identity (responsibility disclaimed)");
519                            None
520                        }
521                    } else if image.is_none()
522                        || image.is_some_and(|image| image.executable.is_none())
523                    {
524                        // A process can exit between try_wait and the kernel
525                        // image read. Empty EOF already acknowledged exec, so
526                        // preserve that module's ordinary exit rather than
527                        // mislabel a disappearing image as trampoline refusal.
528                        // Keep pending in self across await so cancellation does
529                        // not discard validation or reset its original deadline.
530                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531                            Ok(Ok(_status)) => None,
532                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534                        }
535                    } else {
536                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537                    }
538                }
539            },
540        };
541        let pending = self.privacy_exec.take().expect("pending exec");
542        // Exec confirmation describes launch success, not roster retirement.
543        // Keep even a reaped child recorded until its terminal history exists.
544        self.exec.0.finish(if cause.is_some() {
545            crate::activity::ExecState::Refused
546        } else {
547            crate::activity::ExecState::CommittedExec
548        });
549        if let Some(cause) = cause {
550            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551            self.spawn_failure = Some(cause);
552            // No image is admitted on failure. Reach the entire fresh process
553            // group, including a module which spawned a helper before refusal.
554            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556            }
557            let _ = self.child.start_kill();
558        }
559    }
560
561    /// Releases this child's daemon-shutdown roster entry once its exit has
562    /// been recorded. The pid is already reaped and free for reuse, so the
563    /// entry must not outlive the record any longer than that.
564    fn release_roster(&mut self) {
565        self.roster_guard = None;
566    }
567
568    /// Kill the child and, where containment is available, its process tree.
569    ///
570    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
571    /// helper process leaked the helper — the Synapse embedding module's CUDA
572    /// worker holds the GPU allocation, so the leak cost VRAM until the next
573    /// restart of something else. Terminating the job reaches grandchildren that
574    /// a tree walk cannot, including one whose parent has already exited and
575    /// been reparented away.
576    ///
577    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
578    /// direct-child kill still decides the outcome, so containment can never
579    /// change whether a module is reported as stopped.
580    fn start_kill(&mut self) -> io::Result<()> {
581        #[cfg(windows)]
582        if let Some(job) = &self.job {
583            if let Err(error) = job.terminate() {
584                debug!(
585                    error = %error,
586                    "job termination failed; the direct-child kill still owns the outcome"
587                );
588            }
589        }
590        #[cfg(target_os = "linux")]
591        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592        self.child.start_kill()
593    }
594
595    async fn drain_stderr(&mut self, module_id: &str) {
596        if let Some(pump) = self.stdout_pump.take() {
597            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598        }
599
600        let Some(pump) = self.stderr_pump.take() else {
601            return;
602        };
603        settle_stderr_pump(
604            module_id,
605            &self.stderr_ring,
606            pump,
607            STDERR_PUMP_DRAIN_TIMEOUT,
608        )
609        .await;
610    }
611}
612
613/// Bound the restart's wait, not the lifetime of the exited child's stdout
614/// reader. Slow pipe reads or capture writes must not turn the bound into lost
615/// output; dropping the handle lets the reader finish at EOF, like stderr.
616async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617    match timeout(bound, &mut pump).await {
618        Ok(Ok(())) => {}
619        Ok(Err(error)) => {
620            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621        }
622        Err(_) => {
623            drop(pump);
624            warn!(
625                module_id,
626                waited = ?bound,
627                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628            );
629        }
630    }
631}
632
633/// The reader task for one process's stderr, with the ring generation its
634/// lines are attributed to.
635struct StderrPump {
636    task: JoinHandle<()>,
637    generation: u64,
638}
639
640/// Retire an exited process's stderr reader and wait up to `bound` for it to
641/// reach EOF. A reader still running at the bound is detached, not stopped: it
642/// keeps filling the exited process's section of the ring until its pipe
643/// closes, and the tail reads `Incomplete` until then. See
644/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
645async fn settle_stderr_pump(
646    module_id: &str,
647    ring: &Arc<Mutex<StderrRing>>,
648    pump: StderrPump,
649    bound: Duration,
650) {
651    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652    let StderrPump {
653        mut task,
654        generation,
655    } = pump;
656    lock().retire_pump(generation);
657    match timeout(bound, &mut task).await {
658        Ok(Ok(())) => {}
659        Ok(Err(err)) => {
660            let mut ring = lock();
661            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662            ring.finish_pump(generation);
663            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664        }
665        Err(_) => {
666            // Dropping the handle detaches the task; it ends at EOF on its pipe.
667            drop(task);
668            lock().mark_pump_late(
669                generation,
670                format!(
671                    "stderr of the exited process had not reached EOF {bound:?} after it was \
672                     retired (a descendant may still hold the pipe open); lines it still \
673                     writes are kept in that process's section"
674                ),
675            );
676            warn!(
677                module_id,
678                waited = ?bound,
679                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680            );
681        }
682    }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687    EVENTS.get_or_init(|| {
688        let (sender, _receiver) = watch::channel(0);
689        sender
690    })
691}
692
693pub(crate) fn notify_registration_release() {
694    let events = registration_release_events();
695    let next_generation = (*events.borrow()).wrapping_add(1);
696    events.send_replace(next_generation);
697}
698
699/// How to launch one singleton module process.
700#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702    pub module_id: String,
703    pub program: PathBuf,
704    pub args: Vec<String>,
705    pub env: Vec<(String, String)>,
706    /// When true this is a reserved module: each spawn gets a fresh one-time launch
707    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
708    /// process can register this module_id (a security-boundary module like the
709    /// credential vault must not be impersonable while it is down/restarting).
710    pub reserved: bool,
711    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
712    /// Prefixes come from daemon config and must end in `:` before they reach the
713    /// supervisor; the owner module's current spawn nonce authorizes claims under
714    /// each prefix.
715    pub reserved_prefixes: Vec<String>,
716    /// The wire protocol this module speaks, as DECLARED in daemon config.
717    ///
718    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
719    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
720    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
721    /// and NO launch nonce, and a clean exit the daemon did not request is
722    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
723    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
724    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
725    /// because a process ignores an environment variable it does not read.
726    ///
727    /// The argument is the part that cannot be "harmless to a process that
728    /// ignores it": a stock binary exits on an unknown flag before it listens
729    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
730    /// first conformance run against this mode found it. The nonce is withheld
731    /// because a process that will never present it gains nothing from holding
732    /// it, and a secret in the environment of a process that does not need it is
733    /// a leak surface for no benefit.
734    pub protocol: ModuleProtocol,
735    /// Whether two processes of this module may run at once, which is what a
736    /// blue/green swap does for the length of its overlap. Declared in daemon
737    /// config because the daemon must be able to answer it while the module is
738    /// down, and so a module cannot talk itself into it after registering.
739    pub overlap: ModuleOverlap,
740}
741
742/// Only the configured launch inputs enter this value; spawned connection paths,
743/// nonces and inherited environment never do. Its bytes remain daemon-private.
744#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749        f.write_str("LaunchSpecDigest([redacted])")
750    }
751}
752
753impl LaunchSpecDigest {
754    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755        fn field(hash: &mut Sha256, bytes: &[u8]) {
756            hash.update((bytes.len() as u64).to_le_bytes());
757            hash.update(bytes);
758        }
759        let mut hash = Sha256::new();
760        if let Some(program) = spec.program.to_str() {
761            field(&mut hash, b"utf8-program");
762            field(&mut hash, program.as_bytes());
763        } else {
764            // The daemon config is UTF-8, but in-process callers may provide
765            // native paths: lossy conversion would collapse distinct programs.
766            #[cfg(unix)]
767            {
768                use std::os::unix::ffi::OsStrExt;
769                field(&mut hash, b"unix-program");
770                field(&mut hash, spec.program.as_os_str().as_bytes());
771            }
772            #[cfg(windows)]
773            {
774                use std::os::windows::ffi::OsStrExt;
775                field(&mut hash, b"windows-program");
776                let bytes = spec
777                    .program
778                    .as_os_str()
779                    .encode_wide()
780                    .flat_map(u16::to_le_bytes)
781                    .collect::<Vec<_>>();
782                field(&mut hash, &bytes);
783            }
784            #[cfg(not(any(unix, windows)))]
785            field(&mut hash, spec.program.to_string_lossy().as_bytes());
786        }
787        hash.update((spec.args.len() as u64).to_le_bytes());
788        for arg in &spec.args {
789            field(&mut hash, arg.as_bytes());
790        }
791        let mut env = spec.env.iter().collect::<Vec<_>>();
792        env.sort_unstable();
793        hash.update((env.len() as u64).to_le_bytes());
794        for (key, value) in env {
795            field(&mut hash, key.as_bytes());
796            field(&mut hash, value.as_bytes());
797        }
798        field(
799            &mut hash,
800            match spec.protocol {
801                ModuleProtocol::Subc => b"subc",
802                ModuleProtocol::None => b"none",
803            },
804        );
805        Self(hash.finalize().into())
806    }
807}
808
809/// Whether a module tolerates a second process of itself running alongside.
810///
811/// Most modules are single-writer on their store (a WAL, a capture log, a
812/// resident index behind a writer barrier), and two processes on one store
813/// corrupt it. So a swap, which overlaps the old and new process by design,
814/// is refused unless the module's config opts in with `overlap: "safe"`.
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817    /// Never run two processes of this module at once. The default.
818    #[default]
819    Exclusive,
820    /// The module has said a second process of itself is harmless for the
821    /// length of a swap.
822    ///
823    /// Declare it only if a second instance can run for a few seconds without
824    /// touching ANY single-writer store: every database, WAL, index, projector
825    /// and scheduled job the module owns. A lease on part of that state is not
826    /// enough. broca's session lease guards WAL appends while its run index, its
827    /// store projector and its archive fold timer (which unlinks live WAL files)
828    /// stay single-writer, so broca is exclusive despite holding a lease. The
829    /// refusal only fires after this has been decided, so the decision is the
830    /// check.
831    Safe,
832}
833
834impl ModuleOverlap {
835    pub fn as_str(self) -> &'static str {
836        match self {
837            Self::Exclusive => "exclusive",
838            Self::Safe => "safe",
839        }
840    }
841}
842
843/// Environment variable telling a spawned module which case it was started
844/// for, before it sends HELLO. Only a swap candidate carries it, as
845/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
846///
847/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
848/// longer because nobody waits on it, while a plain restart must flip ready
849/// quickly because callers see `module_warming` until it does. Absence means
850/// plain restart, the safe reading. The daemon trusts nothing about it; the
851/// candidate is proven by its launch nonce at HELLO.
852pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
854pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855/// How long a swap waits for its candidate to register and declare itself
856/// ready when the operator does not say. A module warming as a swap candidate
857/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
858/// daemon allows that plus time to start the process and send HELLO.
859pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861/// Bounded restart policy for crash exits.
862///
863/// `max_restarts` is the number of replacement processes allowed after the
864/// initial spawn WITHIN `window`. After that many crash restarts inside one
865/// window the module enters [`ModuleState::Failed`] and the supervisor stops
866/// the crash loop.
867///
868/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
869/// and that only survived because crashes were rare: a module that crashed
870/// three times across a week was disabled forever by crashes that had nothing
871/// to do with each other. That stopped being survivable once modules began
872/// exiting non-zero whenever the daemon's connection to them drops, because
873/// then every daemon-side connection drop spends a unit of the same budget and
874/// one flappy hour permanently stops a healthy module. Restarts older than
875/// `window` release their slot, so a module that crashed twice yesterday has a
876/// full budget today, while a genuine crash loop -- which is fast by
877/// definition -- still reaches the cap and stops.
878#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880    pub max_restarts: u32,
881    /// Base delay before a crash replacement. The actual delay escalates with
882    /// the number of recent crash replacements and is capped by `max_backoff`.
883    pub backoff: Duration,
884    /// Maximum delay before a crash replacement.
885    pub max_backoff: Duration,
886    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
887    /// budget effectively infinite (nothing is ever in-window), which is why
888    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
889    pub window: Duration,
890}
891
892impl RestartPolicy {
893    /// A policy with the default crash window. Callers that care about the
894    /// window say so with [`Self::with_window`]; the ones that do not are
895    /// asking for the standard rate limit, not for no limit.
896    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897        Self {
898            max_restarts,
899            backoff,
900            max_backoff: DEFAULT_MAX_BACKOFF,
901            window: DEFAULT_RESTART_WINDOW,
902        }
903    }
904
905    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906        self.max_backoff = max_backoff;
907        self
908    }
909
910    pub fn with_window(mut self, window: Duration) -> Self {
911        self.window = window;
912        self
913    }
914
915    /// Calculate the capped exponential delay for the next crash replacement.
916    /// `restart_in_window` is zero for the first replacement after an operator
917    /// action (restart, reload, re-enable) cleared the crash ring, or after all
918    /// older crash replacements have aged out of the window.
919    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920        if self.backoff.is_zero() || self.max_backoff.is_zero() {
921            return Duration::ZERO;
922        }
923
924        let mut delay = self.backoff;
925        for _ in 0..restart_in_window {
926            if delay >= self.max_backoff {
927                return self.max_backoff;
928            }
929            delay = delay
930                .checked_mul(10)
931                .unwrap_or(self.max_backoff)
932                .min(self.max_backoff);
933        }
934        delay.min(self.max_backoff)
935    }
936
937    /// The one sentence that explains a budget-exhausted stop, used for both the
938    /// log line and the terminal record so the two cannot drift. It names the
939    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
940    /// exactly what this budget is not.
941    fn budget_exhausted_detail(&self) -> String {
942        format!(
943            "crash budget exhausted: max_restarts={} within window_secs={}",
944            self.max_restarts,
945            self.window.as_secs()
946        )
947    }
948}
949
950impl Default for RestartPolicy {
951    fn default() -> Self {
952        Self {
953            max_restarts: DEFAULT_MAX_RESTARTS,
954            backoff: DEFAULT_BACKOFF,
955            max_backoff: DEFAULT_MAX_BACKOFF,
956            window: DEFAULT_RESTART_WINDOW,
957        }
958    }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963    restart_in_window: u32,
964    delay: Duration,
965}
966
967/// Whether the daemon itself will bring this module back after the exit being
968/// handled: it is enabled AND its in-window crash restarts are below the cap.
969///
970/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
971/// the window are dropped here rather than by a timer, so the count is right
972/// the moment somebody asks and no bookkeeping runs for idle modules.
973fn daemon_will_restart(
974    state: &mut SupervisorSnapshot,
975    policy: &RestartPolicy,
976    now: Instant,
977) -> bool {
978    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988    Report,
989    Restart,
990    Alert,
991}
992
993impl fmt::Display for HealthAction {
994    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995        f.write_str(match self {
996            Self::Report => "report",
997            Self::Restart => "restart",
998            Self::Alert => "alert",
999        })
1000    }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005    /// Optional loopback HTTP endpoint for a managed non-wire process.
1006    /// Changing it applies live on rescan; the process protocol changes only
1007    /// at its next spawn.
1008    pub http: Option<String>,
1009    pub cadence: Duration,
1010    pub deadline: Duration,
1011    pub failure_threshold: u32,
1012    pub on_degraded: HealthAction,
1013    pub on_failing: HealthAction,
1014    pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018    fn default() -> Self {
1019        Self {
1020            http: None,
1021            cadence: DEFAULT_HEALTH_CADENCE,
1022            deadline: DEFAULT_HEALTH_DEADLINE,
1023            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024            on_degraded: HealthAction::Report,
1025            on_failing: HealthAction::Report,
1026            critical: false,
1027        }
1028    }
1029}
1030
1031/// The supervisor's view of one module's health, relayed to clients over
1032/// channel-0 and rendered by `ck health`.
1033///
1034/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1035/// stated here rather than only at the wire type a consumer reads. A reader can
1036/// look up what `None` means; only a writer can silently change it, and the
1037/// writer has no reason to go looking at a downstream contract before editing.
1038///
1039/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1040/// back to `None` on re-registration precisely so a respawned module does not
1041/// carry its predecessor's timestamp — so an old value and an absent one call for
1042/// opposite readings, and anything that defaulted this to a number would make a
1043/// never-probed module indistinguishable from one probed at the epoch.
1044///
1045/// `detail` and `metrics` are `None` when the module published none on this
1046/// probe, which does not mean it reported nothing wrong — it is also the shape
1047/// when the probe never reached it. `last_probe_ms` is what separates those.
1048#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050    pub status: SupervisorHealthStatus,
1051    pub last_probe_ms: Option<u64>,
1052    pub detail: Option<String>,
1053    pub metrics: Option<Value>,
1054    pub consecutive_failures: u32,
1055    /// Number of replies received after a recurring health probe's deadline.
1056    /// Unlike a timeout, every increment proves the module was alive.
1057    pub late_answer_count: u64,
1058    /// End-to-end latency of the newest late reply, measured from probe start.
1059    pub last_late_answer_latency_ms: Option<u64>,
1060    pub last_action: Option<String>,
1061    /// Set together with `last_action`; the pair moves as one, and both being
1062    /// absent means no escalation has ever been taken rather than that the last
1063    /// one succeeded.
1064    pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068    fn default() -> Self {
1069        Self {
1070            status: SupervisorHealthStatus::Unknown,
1071            last_probe_ms: None,
1072            detail: None,
1073            metrics: None,
1074            consecutive_failures: 0,
1075            late_answer_count: 0,
1076            last_late_answer_latency_ms: None,
1077            last_action: None,
1078            last_action_ms: None,
1079        }
1080    }
1081}
1082
1083/// Typed lifecycle state for a supervised module.
1084#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086    Starting,
1087    Running,
1088    Unresponsive,
1089    Restarting,
1090    Draining,
1091    Stopped,
1092    Failed,
1093    Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098        f.write_str(match self {
1099            Self::Starting => "starting",
1100            Self::Running => "running",
1101            Self::Unresponsive => "unresponsive",
1102            Self::Restarting => "restarting",
1103            Self::Draining => "draining",
1104            Self::Stopped => "stopped",
1105            Self::Failed => "failed",
1106            Self::Disabled => "disabled",
1107        })
1108    }
1109}
1110
1111/// Supervisor classification of a child-process exit.
1112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114    Clean,
1115    Crash,
1116    DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120    fn from(kind: ExitKind) -> Self {
1121        match kind {
1122            ExitKind::Clean => Self::Clean,
1123            ExitKind::Crash => Self::Crash,
1124            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125        }
1126    }
1127}
1128
1129/// Exact process identity retained when a supervised module registers its
1130/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133    pub(crate) pid: u32,
1134    pub(crate) start_time: u64,
1135}
1136
1137/// Last observed child exit, if any.
1138#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140    pub kind: ExitKind,
1141    pub code: Option<i32>,
1142    pub signal: Option<i32>,
1143    pub at_ms: u64,
1144}
1145
1146/// Point-in-time module status answerable by subc without forwarding to the
1147/// module process.
1148#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150    pub module_id: String,
1151    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1152    pub waiting_for_previous_pid: Option<u32>,
1153    /// Why the previous instance still prevents a new start.
1154    pub waiting_for_previous_detail: Option<String>,
1155    /// The daemon's Windows per-spawn record: `pipe` after the process it
1156    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1157    /// List/status reports use this record, not the source the module sends
1158    /// in its registration message's manifest.provenance.launch_nonce_source.
1159    pub launch_nonce_source: Option<String>,
1160    pub state: ModuleState,
1161    pub enabled: bool,
1162    pub process_alive: bool,
1163    pub registration_active: bool,
1164    /// The module's declared wire protocol, carried beside `live` because it is
1165    /// what makes `live` readable: the two fields answer one question together.
1166    /// While a process is alive this is its launch declaration, not a later
1167    /// pending-reload edit. When down it is the configured next launch protocol.
1168    pub protocol: ModuleProtocol,
1169    /// Whether the module is serving, under the strongest definition the daemon
1170    /// can assert for its protocol.
1171    ///
1172    /// A subc module must also be REGISTERED: its process being alive says
1173    /// nothing about whether it can take a request. A `protocol: "none"` module
1174    /// never registers, so that term is dropped and this falls back to "enabled,
1175    /// running, and the process the daemon launched is alive" -- which is all
1176    /// the daemon observes about a process that speaks no subc wire. It stays a
1177    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1178    /// rather than printing it bare.
1179    pub live: bool,
1180    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1181    /// restarts have already released their slot, so this count can go down
1182    /// without anybody touching the module.
1183    pub restart_count: u32,
1184    /// Replacement processes spawned over this module's entire supervisor lifetime;
1185    /// unlike `restart_count`, this value is never reset by an operator action
1186    /// and never falls out of a window.
1187    pub lifetime_restarts: u32,
1188    pub spawn_generation: u64,
1189    /// The budget `restart_count` is spent against. Carried alongside the count
1190    /// because the count alone does not say how close the module is to being
1191    /// disabled, and reporting one without the other is what makes an
1192    /// about-to-be-retired module look ordinary.
1193    pub max_restarts: u32,
1194    /// The span `restart_count` is counted over. Carried with the pair above for
1195    /// the same reason they are carried together: "2 of 3" means one thing for a
1196    /// ten-minute window and something else entirely for a lifetime.
1197    pub restart_window: Duration,
1198    /// Effective drain and restart timing policy used by this running module.
1199    /// These values are carried together with the restart budget so status
1200    /// readers can compare configured intent with what the supervisor applied.
1201    pub drain_timeout: Duration,
1202    pub restart_backoff: Duration,
1203    pub restart_max_backoff: Duration,
1204    /// The module's process. On macOS this stays absent while the `ck-subc`
1205    /// launch trampoline is still running in that pid, and appears once the
1206    /// exec acknowledgement confirms the module image has replaced it. Launch
1207    /// time and the supervisor's own hold on the process are unaffected.
1208    pub pid: Option<u32>,
1209    pub spawned_at_ms: Option<u64>,
1210    pub spawned_from: Option<PathBuf>,
1211    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212    pub process_start_time: Option<u64>,
1213    pub last_exit: Option<ExitReport>,
1214    pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220    generation: u64,
1221    turn: u64,
1222    registered_connection: Option<ConnectionId>,
1223    next_probe_at: Option<Instant>,
1224    wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230    #[cfg(windows)]
1231    pipe_consumed: Option<Arc<OnceLock<()>>>,
1232    /// Counts running-child loop turns, not executor polls of a parked wait.
1233    #[cfg(test)]
1234    actor_turns: u64,
1235    /// Running select after exec confirmation and with no unconsumed registry
1236    /// event. Recording a spawn or starting a loop turn is not this barrier.
1237    #[cfg(test)]
1238    actor_select: Option<ActorSelectCheckpoint>,
1239    state: ModuleState,
1240    enabled: bool,
1241    process_alive: bool,
1242    spawned_protocol: Option<ModuleProtocol>,
1243    spawn_failure: Option<String>,
1244    waiting_for_previous_pid: Option<u32>,
1245    waiting_for_previous_detail: Option<String>,
1246    /// When each crash restart was spent, oldest first. This IS the crash
1247    /// budget: its in-window length is the count an operator sees and the count
1248    /// the restart decision is made against, so there is no second counter that
1249    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1250    /// operator actions that used to zero the old lifetime counter.
1251    crash_restarts: VecDeque<Instant>,
1252    lifetime_restarts: u32,
1253    /// Successful child spawns in this daemon incarnation.
1254    ///
1255    /// `lifetime_restarts` was considered and rejected: it starts at zero
1256    /// (line 640), successful initial/operator spawns in `set_running` do not
1257    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1258    /// increments before a successful replacement exists (lines 604, 3846,
1259    /// and 3921), so a failed spawn can consume it. This counter moves only
1260    /// when a live PID is accepted below.
1261    spawn_generation: u64,
1262    pid: Option<u32>,
1263    #[cfg(target_os = "macos")]
1264    report_ready: Option<Arc<OnceLock<()>>>,
1265    /// Last reaped child, retained after current process facts are cleared.
1266    reaped_pid: Option<u32>,
1267    /// Whether a replacement has already been scheduled for the current process.
1268    respawn_pending: bool,
1269    /// A second restart is waiting for the replacement already scheduled.
1270    coalesced_restart_pending: bool,
1271    spawned_at_ms: Option<u64>,
1272    spawned_from: Option<PathBuf>,
1273    launch_spec_digest: Option<LaunchSpecDigest>,
1274    spawned_file_identity: Option<SpawnedFileIdentity>,
1275    process_start_time: Option<u64>,
1276    deliberate_severance: Option<ProcessIdentity>,
1277    last_exit: Option<ExitReport>,
1278    /// Diagnostic attached to the next drain's terminal record, if any.
1279    drain_disposition_detail: Option<String>,
1280    health: ModuleHealthStatus,
1281    /// Whether the current process was started as a swap candidate and so
1282    /// lives in the module's alternate cgroup. The next swap's candidate takes
1283    /// the other one, so the two processes of a swap never share a cgroup. A
1284    /// plain spawn always uses the primary cgroup.
1285    in_alternate_slot: bool,
1286    /// Whether the current `Draining` state ends in a replacement process
1287    /// (restart, reload, health restart) rather than a stop. Only meaningful
1288    /// while `state` is `Draining`; every entry into that state rewrites it.
1289    /// It is what lets route.open answer the retryable `module_reloading` to a
1290    /// consumer that reaches a still-registered process mid-restart, instead of
1291    /// the `supervisor_not_live` a stop or disable deserves.
1292    draining_to_replace: bool,
1293    /// Whether a configuration update has been applied since the current
1294    /// process was spawned, so that process runs an older spec than the one
1295    /// the supervisor now holds. A queued restart is only coalesced into a
1296    /// fresher process when this is false: a restart requested to pick up a
1297    /// new configuration must not be satisfied by a process that predates it.
1298    configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302    /// The pid that status, provenance and resource readings may report. While
1303    /// the launch trampoline still runs in the pid, reading its executable or
1304    /// resource use would describe `ck-subc`, not the module, so none is
1305    /// reported until the exec acknowledgement confirms the module image.
1306    fn reported_pid(&self) -> Option<u32> {
1307        #[cfg(target_os = "macos")]
1308        if self
1309            .report_ready
1310            .as_ref()
1311            .is_some_and(|ready| ready.get().is_none())
1312        {
1313            return None;
1314        }
1315        self.pid
1316    }
1317
1318    fn starting() -> Self {
1319        Self::new(ModuleState::Starting, true)
1320    }
1321
1322    fn disabled() -> Self {
1323        Self::new(ModuleState::Disabled, false)
1324    }
1325
1326    /// Crash restarts still inside `window`, having dropped the ones that are
1327    /// not. Pruning on read is what makes the budget a rate: an instant older
1328    /// than the window stops holding a slot the moment anybody counts.
1329    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330        while let Some(oldest) = self.crash_restarts.front() {
1331            if now.duration_since(*oldest) > window {
1332                self.crash_restarts.pop_front();
1333            } else {
1334                break;
1335            }
1336        }
1337        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338    }
1339
1340    /// Spend one unit of the crash budget and record the restart in the ledger.
1341    ///
1342    /// The ring is bounded by the cap because more than `max_restarts` in-window
1343    /// instants can never be reached (the caller refuses the restart first), so
1344    /// anything beyond that is an unbounded queue waiting to happen.
1345    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346        self.crash_restarts.push_back(now);
1347        while self.crash_restarts.len() > policy.max_restarts as usize {
1348            self.crash_restarts.pop_front();
1349        }
1350        self.lifetime_restarts += 1;
1351    }
1352
1353    /// Reserve one crash-restart slot and calculate the delay before respawning.
1354    /// The count is captured before recording this restart, so the first retry
1355    /// uses the base delay and each later in-window retry escalates once.
1356    fn next_crash_restart(
1357        &mut self,
1358        policy: &RestartPolicy,
1359        now: Instant,
1360    ) -> Option<CrashRestartSchedule> {
1361        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362        if restart_in_window >= policy.max_restarts {
1363            return None;
1364        }
1365        self.record_crash_restart(policy, now);
1366        Some(CrashRestartSchedule {
1367            restart_in_window,
1368            delay: policy.delay_for_restart(restart_in_window),
1369        })
1370    }
1371
1372    /// Give the module its full budget back, as an operator restart, reload, or
1373    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1374    /// ledger of what actually happened, and an operator action does not unmake
1375    /// the crashes.
1376    fn clear_crash_restarts(&mut self) {
1377        self.crash_restarts.clear();
1378    }
1379
1380    fn new(state: ModuleState, enabled: bool) -> Self {
1381        Self {
1382            exec_outcome: None,
1383            #[cfg(test)]
1384            actor_turns: 0,
1385            #[cfg(test)]
1386            actor_select: None,
1387            state,
1388            enabled,
1389            process_alive: false,
1390            #[cfg(windows)]
1391            pipe_consumed: None,
1392            spawned_protocol: None,
1393            spawn_failure: None,
1394            waiting_for_previous_pid: None,
1395            waiting_for_previous_detail: None,
1396            crash_restarts: VecDeque::new(),
1397            lifetime_restarts: 0,
1398            spawn_generation: 0,
1399            pid: None,
1400            #[cfg(target_os = "macos")]
1401            report_ready: None,
1402            reaped_pid: None,
1403            respawn_pending: false,
1404            coalesced_restart_pending: false,
1405            spawned_at_ms: None,
1406            spawned_from: None,
1407            launch_spec_digest: None,
1408            spawned_file_identity: None,
1409            process_start_time: None,
1410            deliberate_severance: None,
1411            last_exit: None,
1412            drain_disposition_detail: None,
1413            health: ModuleHealthStatus::default(),
1414            in_alternate_slot: false,
1415            draining_to_replace: false,
1416            configuration_updated_since_spawn: false,
1417        }
1418    }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427    version: u8,
1428    frames: mpsc::Sender<Frame>,
1429    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1430    /// from which event. The full frame channel cannot carry that news, so it
1431    /// travels beside it; see `SpawnEventFeed::subscribe`.
1432    lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437    daemon_incarnation: String,
1438    seq: u64,
1439    capacity: usize,
1440    live: HashMap<String, LiveSpawn>,
1441    generations: HashMap<String, u64>,
1442    events: VecDeque<SpawnEvent>,
1443    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447    fn default() -> Self {
1448        Self {
1449            daemon_incarnation: "unconfigured".to_string(),
1450            seq: 0,
1451            capacity: SPAWN_EVENT_RING_CAPACITY,
1452            live: HashMap::new(),
1453            generations: HashMap::new(),
1454            events: VecDeque::new(),
1455            subscribers: HashMap::new(),
1456        }
1457    }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462    Arc<Mutex<SpawnEventState>>,
1463    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468    ForeignIncarnation { current: String },
1469    TooOld { oldest: SpawnCursor },
1470    Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474    fn configure_incarnation(&self, daemon_incarnation: String) {
1475        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476        state.daemon_incarnation = daemon_incarnation;
1477        state.seq = 0;
1478        state.live.clear();
1479        state.generations.clear();
1480        state.events.clear();
1481        state.subscribers.clear();
1482    }
1483
1484    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485        SpawnCursor {
1486            daemon_incarnation: state.daemon_incarnation.clone(),
1487            seq: state.seq,
1488        }
1489    }
1490
1491    fn snapshot(&self) -> SpawnSnapshot {
1492        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495        SpawnSnapshot {
1496            cursor: Self::cursor(&state),
1497            ring_bound: state.capacity as u64,
1498            live,
1499        }
1500    }
1501
1502    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504        let generation = state
1505            .generations
1506            .get(module_id)
1507            .copied()
1508            .unwrap_or(0)
1509            .checked_add(1)
1510            .expect("spawn generation exhausted");
1511        state.generations.insert(module_id.to_string(), generation);
1512        let live = LiveSpawn {
1513            module_id: module_id.to_string(),
1514            spawn_generation: generation,
1515            pid,
1516            spawned_at_ms,
1517        };
1518        state.live.insert(module_id.to_string(), live);
1519        Self::emit_locked(
1520            &mut state,
1521            SpawnEventKind::Spawned,
1522            module_id.to_string(),
1523            generation,
1524            pid,
1525            None,
1526            None,
1527        );
1528        generation
1529    }
1530
1531    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533        let Some(live) = state.live.remove(module_id) else {
1534            warn!(
1535                module_id,
1536                "terminal record had no live spawn event identity"
1537            );
1538            return;
1539        };
1540        Self::emit_locked(
1541            &mut state,
1542            SpawnEventKind::Exited,
1543            module_id.to_string(),
1544            live.spawn_generation,
1545            live.pid,
1546            exit_code,
1547            exit_signal,
1548        );
1549    }
1550
1551    /// Report the exit of a process that a swap has already replaced.
1552    ///
1553    /// `emit_exited` removes the module's live entry, which after a swap's
1554    /// cutover describes the promoted candidate, not the old process now
1555    /// exiting. This emits the old generation's exit and leaves the live entry
1556    /// alone unless it still names that generation.
1557    fn emit_superseded_exited(
1558        &self,
1559        module_id: &str,
1560        spawn_generation: u64,
1561        pid: u32,
1562        exit_code: Option<i32>,
1563        exit_signal: Option<i32>,
1564    ) {
1565        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566        if state
1567            .live
1568            .get(module_id)
1569            .is_some_and(|live| live.spawn_generation == spawn_generation)
1570        {
1571            state.live.remove(module_id);
1572        }
1573        Self::emit_locked(
1574            &mut state,
1575            SpawnEventKind::Exited,
1576            module_id.to_string(),
1577            spawn_generation,
1578            pid,
1579            exit_code,
1580            exit_signal,
1581        );
1582    }
1583
1584    #[allow(clippy::too_many_arguments)]
1585    fn emit_locked(
1586        state: &mut SpawnEventState,
1587        kind: SpawnEventKind,
1588        module_id: String,
1589        spawn_generation: u64,
1590        pid: u32,
1591        exit_code: Option<i32>,
1592        exit_signal: Option<i32>,
1593    ) {
1594        state.seq = state
1595            .seq
1596            .checked_add(1)
1597            .expect("spawn event sequence exhausted");
1598        let event = SpawnEvent {
1599            cursor: Self::cursor(state),
1600            kind,
1601            module_id,
1602            spawn_generation,
1603            pid,
1604            exit_code,
1605            exit_signal,
1606        };
1607        state.events.push_back(event.clone());
1608        while state.events.len() > state.capacity {
1609            state.events.pop_front();
1610        }
1611        let body = match serde_json::to_vec(&event) {
1612            Ok(body) => body,
1613            Err(error) => {
1614                error!(%error, "failed to serialize supervisor spawn event");
1615                return;
1616            }
1617        };
1618        state.subscribers.retain(|(connection_id, corr), subscriber| {
1619            let frame = Frame::build_with_version(
1620                subscriber.version,
1621                FrameType::StreamData,
1622                control_flags(),
1623                0,
1624                0,
1625                *corr,
1626                body.clone(),
1627            );
1628            match frame {
1629                Ok(frame) => {
1630                    if subscriber.frames.try_send(frame).is_ok() {
1631                        true
1632                    } else {
1633                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634                        if let Some(lagged) = subscriber.lagged.take() {
1635                            let _ = lagged.send(event.cursor.clone());
1636                        }
1637                        false
1638                    }
1639                }
1640                Err(error) => {
1641                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642                    false
1643                }
1644            }
1645        });
1646    }
1647
1648    fn subscribe(
1649        &self,
1650        connection_id: ConnectionId,
1651        corr: u64,
1652        version: u8,
1653        since: Option<SpawnCursor>,
1654        sink: FrameSink,
1655    ) -> Result<(), SpawnSubscribeRefusal> {
1656        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658        {
1659            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660            let replay = if let Some(since) = since {
1661                if since.daemon_incarnation != state.daemon_incarnation {
1662                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663                        current: state.daemon_incarnation.clone(),
1664                    });
1665                }
1666                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667                    if since.seq < oldest.seq.saturating_sub(1) {
1668                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669                    }
1670                }
1671                state
1672                    .events
1673                    .iter()
1674                    .filter(|event| event.cursor.seq > since.seq)
1675                    .cloned()
1676                    .collect::<Vec<_>>()
1677            } else {
1678                Vec::new()
1679            };
1680            for event in replay {
1681                let body = serde_json::to_vec(&event)
1682                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683                let frame = Frame::build_with_version(
1684                    version,
1685                    FrameType::StreamData,
1686                    control_flags(),
1687                    0,
1688                    0,
1689                    corr,
1690                    body,
1691                )
1692                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693                frames
1694                    .try_send(frame)
1695                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696            }
1697            state.subscribers.insert(
1698                (connection_id, corr),
1699                SpawnSubscriber {
1700                    version,
1701                    frames,
1702                    lagged: Some(lagged),
1703                },
1704            );
1705        }
1706        // The lagged terminal is sent here, by the forwarder, rather than by
1707        // the emitter: at the moment of the drop the subscriber's own channel
1708        // is full, and writing to the connection sink directly from the emitter
1709        // would put the Error AHEAD of the events still queued in that channel
1710        // (and the emitter holds the feed lock, so it cannot await the sink).
1711        // Dropping the subscriber drops the only sender, so `recv` drains every
1712        // queued event and then returns `None`; only then is the Error sent, so
1713        // the client sees each event it can keep, then the reason it was cut.
1714        // Cancel and connection removal drop the oneshot unsent, so they end
1715        // the stream with no Error.
1716        tokio::spawn(async move {
1717            while let Some(frame) = receiver.recv().await {
1718                if sink.send(frame).await.is_err() {
1719                    return;
1720                }
1721            }
1722            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723                return;
1724            };
1725            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726                Ok(frame) => {
1727                    let _ = sink.send(frame).await;
1728                }
1729                Err(error) => {
1730                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731                }
1732            }
1733        });
1734        Ok(())
1735    }
1736
1737    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738        let Some(subscriber) = self
1739            .0
1740            .lock()
1741            .unwrap_or_else(|p| p.into_inner())
1742            .subscribers
1743            .remove(&(connection_id, corr))
1744        else {
1745            return false;
1746        };
1747        if let Ok(frame) = Frame::build_with_version(
1748            subscriber.version,
1749            FrameType::StreamEnd,
1750            control_flags(),
1751            0,
1752            0,
1753            corr,
1754            Vec::new(),
1755        ) {
1756            tokio::spawn(async move {
1757                let _ = subscriber.frames.send(frame).await;
1758            });
1759        }
1760        true
1761    }
1762
1763    fn remove_connection(&self, connection_id: ConnectionId) {
1764        self.0
1765            .lock()
1766            .unwrap_or_else(|p| p.into_inner())
1767            .subscribers
1768            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769    }
1770
1771    #[cfg(any(test, feature = "test-support"))]
1772    fn set_capacity(&self, capacity: usize) {
1773        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774    }
1775
1776    #[cfg(any(test, feature = "test-support"))]
1777    fn subscriber_count(&self) -> usize {
1778        self.0
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .subscribers
1782            .len()
1783    }
1784}
1785
1786/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1787/// The terminal Error a lagged spawn subscriber receives after its queued events.
1788fn spawn_subscriber_lagged_frame(
1789    version: u8,
1790    corr: u64,
1791    first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796            .to_string(),
1797        detail: Some(serde_json::json!({
1798            "first_undelivered_cursor": first_undelivered
1799        })),
1800    })
1801    .map_err(|error| error.to_string())?;
1802    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803        .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807    fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809    /// Whether the supervisor is replacing this module's process right now: an
1810    /// operator restart or reload, a health restart, or a crash respawn whose
1811    /// backoff is running. A module in that state is not live, but a consumer
1812    /// refused now should retry shortly rather than treat the target as gone.
1813    /// Stopped, failed, and disabled modules are not replacing.
1814    fn process_replacing(&self, _module_id: &str) -> bool {
1815        false
1816    }
1817}
1818
1819/// Shared process-liveness registry keyed by supervised `module_id`.
1820#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826    pub fn new() -> Self {
1827        Self::default()
1828    }
1829
1830    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831        let mut snapshots = self
1832            .snapshots
1833            .lock()
1834            .unwrap_or_else(|poisoned| poisoned.into_inner());
1835        snapshots.insert(module_id, snapshot);
1836    }
1837
1838    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        let is_current = snapshots
1844            .get(module_id)
1845            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846            .unwrap_or(false);
1847        if is_current {
1848            snapshots.remove(module_id);
1849        }
1850    }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854    fn process_live(&self, module_id: &str) -> Option<bool> {
1855        let snapshot = {
1856            let snapshots = self
1857                .snapshots
1858                .lock()
1859                .unwrap_or_else(|poisoned| poisoned.into_inner());
1860            snapshots.get(module_id).cloned()
1861        }?;
1862        let snapshot = snapshot
1863            .lock()
1864            .unwrap_or_else(|poisoned| poisoned.into_inner());
1865        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866    }
1867
1868    fn process_replacing(&self, module_id: &str) -> bool {
1869        let Some(snapshot) = self
1870            .snapshots
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner())
1873            .get(module_id)
1874            .cloned()
1875        else {
1876            return false;
1877        };
1878        let snapshot = snapshot
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner());
1881        snapshot.enabled
1882            && match snapshot.state {
1883                ModuleState::Restarting => true,
1884                ModuleState::Draining => snapshot.draining_to_replace,
1885                ModuleState::Starting
1886                | ModuleState::Running
1887                | ModuleState::Unresponsive
1888                | ModuleState::Stopped
1889                | ModuleState::Failed
1890                | ModuleState::Disabled => false,
1891            }
1892    }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898    reached: tokio::sync::Notify,
1899    resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904    Spawn,
1905    Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910    deadline: Instant,
1911    kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918    start_requester: Option<Principal>,
1919    pending_activity_restart: bool,
1920    provenance_probe: ExecutableIdentityProbe,
1921    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1922    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923    /// A reload acknowledges completion only after its replacement registers.
1924    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925    restart_policy: RestartPolicy,
1926    /// This module's RESOLVED drain budget: per-module config when present,
1927    /// else `default_drain_timeout`.
1928    drain_timeout: Duration,
1929    /// Shared with the status handle so the attested value changes atomically
1930    /// when a rescan updates the running drain policy.
1931    effective_drain_timeout: Arc<Mutex<Duration>>,
1932    /// The supervisor-wide fallback, kept so a configuration update that
1933    /// REMOVES the per-module override can re-resolve to it.
1934    default_drain_timeout: Duration,
1935    health: HealthConfig,
1936    connection_file_path: Option<PathBuf>,
1937    capture_logs_dir: Option<PathBuf>,
1938    forwarding: Option<Arc<ForwardingTable>>,
1939    /// The shared handle, so every spawn path (initial, restart, reload) records the
1940    /// reserved-module launch nonce the HELLO verifier checks against.
1941    supervisor_handle: Option<SupervisorHandle>,
1942    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1943    /// status queries.
1944    ///
1945    /// One ring per module, held across every respawn. The lines explaining an exit
1946    /// are written BEFORE that exit, so a ring recreated per process would be empty
1947    /// exactly when it is asked for.
1948    stderr_ring: Arc<Mutex<StderrRing>>,
1949    terminal_ring: Arc<Mutex<TerminalRing>>,
1950    spawn_events: SpawnEventFeed,
1951    child_roster: ChildRoster,
1952    #[cfg(target_os = "linux")]
1953    cgroup_placement: Option<subc_cgroup::Placement>,
1954    #[cfg(test)]
1955    test_seed_stale_facts_before_enable_spawn: bool,
1956    #[cfg(test)]
1957    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962    spec: ModuleSpec,
1963    health: HealthConfig,
1964}
1965
1966/// Shared daemon lookup table for supervised module handles.
1967///
1968/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1969/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1970/// launch nonces recorded at spawn are checked by the same daemon instance.
1971#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976    /// Module ids the supervisor has taken on. An id is added BEFORE the
1977    /// module's first process is spawned and removed only when the module
1978    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1979    /// the keys of `modules`.
1980    ///
1981    /// `modules` cannot answer "is this module configured?" on its own: a
1982    /// [`SupervisedModule`] only exists once its process has been spawned, and
1983    /// a fast child can connect, register, sync its scopes and ask about them
1984    /// before the supervisor has inserted it. Answering "not configured" in that
1985    /// gap makes scope admission refuse with the terminal "will never sync"
1986    /// instead of the retryable "has not synced yet".
1987    configured_ids: Arc<Mutex<HashSet<String>>>,
1988    spawn_events: SpawnEventFeed,
1989    /// The current expected launch nonce for each reserved module_id. Set when the
1990    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1991    /// non-reserved module never has an entry here and is never nonce-checked.
1992    /// Reserved module ids and the nonce that authorizes their next HELLO.
1993    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1994    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1995    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1996    /// had NO entry and admitted anyone: the reservation protected the nonce
1997    /// holder, not the NAME (found live by CKCRED's canary probe registering
1998    /// against a reserved scratch id).
1999    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2001    ///
2002    /// This is deliberately in-memory only: subc is state-free across daemon
2003    /// restarts, and the tombstone only explains the hours-after-removal window
2004    /// while this executing daemon is still alive. Do not persist it in a store.
2005    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006    /// The current launch nonce for every supervised spawn. This is separate from
2007    /// reserved_nonces because consumer route.open attestation applies to all spawned
2008    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2009    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010    /// Reserved namespace prefixes mapped to the supervised owner module whose
2011    /// current spawn nonce authorizes HELLO claims below the prefix.
2012    ///
2013    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2014    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2015    /// accidental collisions and lower-trust processes from squatting protected
2016    /// namespaces.
2017    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018    /// Blue/green swaps in progress, by module id. An entry exists from just
2019    /// before the candidate process is spawned until the swap has failed, or
2020    /// has cut over and the old process is gone. While it exists, HELLO for the
2021    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2022    /// consumer attestation accepts both processes' nonces.
2023    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2025    promotion_observer: PromotionObserverSlot,
2026    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2027    /// this daemon-wide ordering, a rescan could retire or update a module while a
2028    /// concurrent reload still held its old handle and launch specification.
2029    operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032/// Told when a swap has promoted its candidate to be the module's active
2033/// registration.
2034///
2035/// An ordinary HELLO runs the control plane's registration side effects (the
2036/// capability cache, the deny census, the requirement recompute) as it
2037/// registers. A swap candidate's HELLO does not, because it is not routable;
2038/// promotion is when those must run instead, and promotion happens in the
2039/// supervisor, which has no other way into the control handler.
2040pub(crate) trait SwapPromotionObserver: Send + Sync {
2041    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2045/// control handler) owns this handle, so a strong reference back would be a
2046/// cycle that keeps both alive.
2047#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052        f.write_str("PromotionObserverSlot")
2053    }
2054}
2055
2056/// The nonces of one open swap.
2057#[derive(Debug, Clone)]
2058struct OpenSwap {
2059    /// The launch nonce minted for the candidate process. It is the swap
2060    /// token: the only thing that admits a HELLO into the candidate slot.
2061    candidate_nonce: String,
2062    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2063    /// here because cutover moves the module's recorded spawn nonce to the
2064    /// candidate while the incumbent is still draining and its consumers are
2065    /// still attesting with this one.
2066    incumbent_nonce: Option<String>,
2067    /// Set once a HELLO has been admitted with the swap token, so the token
2068    /// admits one registration and cannot be replayed after cutover empties
2069    /// the candidate slot.
2070    candidate_admitted: bool,
2071}
2072
2073/// What the swap gate says about a HELLO. See
2074/// [`SupervisorHandle::swap_hello_admission`].
2075#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077    /// No swap is open for the id (or the HELLO carries the incumbent's own
2078    /// nonce); the ordinary gates decide.
2079    NotSwapping,
2080    /// The HELLO carries the swap token: register it into the candidate slot.
2081    Candidate,
2082    /// A swap is open and the HELLO carries a nonce the supervisor did not
2083    /// mint for this id, no nonce, or a token already used.
2084    Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089    Exact {
2090        module_id: String,
2091    },
2092    Prefix {
2093        prefix: String,
2094        owner_module_id: String,
2095    },
2096}
2097
2098impl SupervisorHandle {
2099    pub fn new() -> Self {
2100        Self::default()
2101    }
2102
2103    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104        self.spawn_events.snapshot()
2105    }
2106
2107    pub(crate) fn subscribe_spawns(
2108        &self,
2109        connection_id: ConnectionId,
2110        corr: u64,
2111        version: u8,
2112        since: Option<SpawnCursor>,
2113        sink: FrameSink,
2114    ) -> Result<(), SpawnSubscribeRefusal> {
2115        self.spawn_events
2116            .subscribe(connection_id, corr, version, since, sink)
2117    }
2118
2119    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120        self.spawn_events.cancel(connection_id, corr)
2121    }
2122
2123    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124        self.spawn_events.remove_connection(connection_id);
2125    }
2126
2127    #[cfg(any(test, feature = "test-support"))]
2128    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129        assert!(capacity > 0, "spawn event capacity must be non-zero");
2130        self.spawn_events.set_capacity(capacity);
2131    }
2132
2133    #[cfg(any(test, feature = "test-support"))]
2134    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135        self.spawn_events.subscriber_count()
2136    }
2137
2138    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2139    /// a respawn invalidates stale consumer identities.
2140    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141        self.spawn_nonces
2142            .lock()
2143            .unwrap_or_else(|poisoned| poisoned.into_inner())
2144            .insert(module_id.to_string(), nonce);
2145    }
2146
2147    /// Record the launch nonce expected from the next HELLO for a reserved module,
2148    /// replacing any prior nonce (a respawn invalidates the previous one).
2149    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150        self.reserved_nonces
2151            .lock()
2152            .unwrap_or_else(|poisoned| poisoned.into_inner())
2153            .insert(module_id.to_string(), Some(nonce));
2154    }
2155
2156    /// Record namespace prefixes owned by a supervised module.
2157    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158        let mut owners = self
2159            .reserved_prefix_owners
2160            .lock()
2161            .unwrap_or_else(|poisoned| poisoned.into_inner());
2162        owners.retain(|_, owner| owner != owner_module_id);
2163        for prefix in prefixes {
2164            owners.insert(prefix.clone(), owner_module_id.to_string());
2165        }
2166    }
2167
2168    /// The launch nonce most recently minted for a module's spawn, if any.
2169    #[cfg(test)]
2170    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171        self.spawn_nonces
2172            .lock()
2173            .unwrap_or_else(|poisoned| poisoned.into_inner())
2174            .get(module_id)
2175            .cloned()
2176    }
2177
2178    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180        let spawn_nonce = self
2181            .spawn_nonces
2182            .lock()
2183            .unwrap_or_else(|poisoned| poisoned.into_inner())
2184            .get(&spec.module_id)
2185            .cloned();
2186        let mut reserved_nonces = self
2187            .reserved_nonces
2188            .lock()
2189            .unwrap_or_else(|poisoned| poisoned.into_inner());
2190        if spec.reserved {
2191            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2192            // reserved name whose module has never spawned has no legitimate
2193            // holder, and the entry's absence is what used to leave the name
2194            // open to the first claimant.
2195            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196        }
2197        drop(reserved_nonces);
2198        // A later unreserved declaration must not silently unreserve an id that
2199        // was retained after its reserved configuration was removed. The explicit
2200        // release ceremony is the only operation that retires that gate.
2201        self.removal_tombstones
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .remove(&spec.module_id);
2205    }
2206
2207    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2208    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2209    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2210    /// with no matching prefix are always authorized.
2211    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212        self.reserved_hello_rejection(module_id, presented)
2213            .is_none()
2214    }
2215
2216    pub(crate) fn reserved_hello_rejection(
2217        &self,
2218        module_id: &str,
2219        presented: Option<&str>,
2220    ) -> Option<ReservedHelloRejection> {
2221        let nonces = self
2222            .reserved_nonces
2223            .lock()
2224            .unwrap_or_else(|poisoned| poisoned.into_inner());
2225        if let Some(expected) = nonces.get(module_id) {
2226            // `None` = reserved with no legitimate holder: refuse every
2227            // presentation, because no process can hold a nonce that was never
2228            // minted. Only a real minted nonce admits, in constant time.
2229            let authorized = match expected {
2230                Some(expected) => {
2231                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232                }
2233                None => false,
2234            };
2235            if authorized {
2236                return None;
2237            }
2238            return Some(ReservedHelloRejection::Exact {
2239                module_id: module_id.to_string(),
2240            });
2241        }
2242        drop(nonces);
2243
2244        let matched_prefix = self
2245            .reserved_prefix_owners
2246            .lock()
2247            .unwrap_or_else(|poisoned| poisoned.into_inner())
2248            .iter()
2249            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250            .max_by_key(|(prefix, _)| prefix.len())
2251            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252        let (prefix, owner_module_id) = matched_prefix?;
2253
2254        let authorized = presented.is_some_and(|presented| {
2255            self.spawn_nonces
2256                .lock()
2257                .unwrap_or_else(|poisoned| poisoned.into_inner())
2258                .get(&owner_module_id)
2259                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260                // While the owner is being swapped, children started by
2261                // either of its two processes hold that process's nonce.
2262                || self.swap_nonce_matches(&owner_module_id, presented)
2263        });
2264        if authorized {
2265            None
2266        } else {
2267            Some(ReservedHelloRejection::Prefix {
2268                prefix,
2269                owner_module_id,
2270            })
2271        }
2272    }
2273
2274    /// Whether a consumer connection proved it came from a daemon-spawned module.
2275    ///
2276    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2277    /// accepted only for module ids the supervisor has spawned.
2278    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279        if presented.is_empty() {
2280            return false;
2281        }
2282        let nonces = self
2283            .spawn_nonces
2284            .lock()
2285            .unwrap_or_else(|poisoned| poisoned.into_inner());
2286        let current = nonces
2287            .get(module_id)
2288            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289        drop(nonces);
2290        // During a swap two processes of the module are alive, and a consumer
2291        // started by either one presents that process's nonce. Accepting only
2292        // the recorded one would fail the incumbent's consumers for the whole
2293        // overlap once cutover moves the record to the candidate.
2294        current || self.swap_nonce_matches(module_id, presented)
2295    }
2296
2297    /// Whether `presented` is either nonce of an open swap for `module_id`.
2298    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299        let swaps = self
2300            .swaps
2301            .lock()
2302            .unwrap_or_else(|poisoned| poisoned.into_inner());
2303        swaps.get(module_id).is_some_and(|swap| {
2304            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307                })
2308        })
2309    }
2310
2311    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2312    /// Called before the candidate process exists.
2313    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314        let incumbent_nonce = self
2315            .spawn_nonces
2316            .lock()
2317            .unwrap_or_else(|poisoned| poisoned.into_inner())
2318            .get(module_id)
2319            .cloned();
2320        self.swaps
2321            .lock()
2322            .unwrap_or_else(|poisoned| poisoned.into_inner())
2323            .insert(
2324                module_id.to_string(),
2325                OpenSwap {
2326                    candidate_nonce,
2327                    incumbent_nonce,
2328                    candidate_admitted: false,
2329                },
2330            );
2331    }
2332
2333    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2334    /// the module's recorded one.
2335    pub(crate) fn close_swap(&self, module_id: &str) {
2336        self.swaps
2337            .lock()
2338            .unwrap_or_else(|poisoned| poisoned.into_inner())
2339            .remove(module_id);
2340    }
2341
2342    /// Install the observer told about swap promotions, replacing any earlier
2343    /// one.
2344    pub(crate) fn set_swap_promotion_observer(
2345        &self,
2346        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347    ) {
2348        *self
2349            .promotion_observer
2350            .0
2351            .lock()
2352            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353    }
2354
2355    /// Tell the installed observer, if it is still alive, that a swap promoted
2356    /// `registration`.
2357    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358        let observer = self
2359            .promotion_observer
2360            .0
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner())
2363            .as_ref()
2364            .and_then(std::sync::Weak::upgrade);
2365        if let Some(observer) = observer {
2366            observer.swap_promoted(registration);
2367        }
2368    }
2369
2370    /// Whether a swap is open for `module_id`.
2371    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372        self.swaps
2373            .lock()
2374            .unwrap_or_else(|poisoned| poisoned.into_inner())
2375            .contains_key(module_id)
2376    }
2377
2378    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2379    /// respawn would, once cutover has made the candidate the module's process.
2380    /// The swap stays open so the incumbent's nonce keeps attesting until the
2381    /// incumbent has drained and exited.
2382    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383        let candidate_nonce = self
2384            .swaps
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .get(module_id)
2388            .map(|swap| swap.candidate_nonce.clone());
2389        let Some(nonce) = candidate_nonce else {
2390            return;
2391        };
2392        self.set_spawn_nonce(module_id, nonce.clone());
2393        if reserved {
2394            self.set_reserved_nonce(module_id, nonce);
2395        }
2396    }
2397
2398    /// The swap gate for a HELLO claiming `module_id`.
2399    ///
2400    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2401    /// presents the candidate nonce, which the reserved gate (holding the
2402    /// incumbent's nonce) would refuse as `reserved_module` before swap
2403    /// admission was ever reached. And it applies to unreserved ids too: for an
2404    /// unreserved id the only thing that ever stopped a second process claiming
2405    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2406    /// refusal a swap lifts for its candidate.
2407    ///
2408    /// The incumbent's own nonce falls through to the ordinary gates, which
2409    /// treat it as they always have (a live incumbent is refused as a
2410    /// duplicate). Anything else while a swap is open is refused, including an
2411    /// absent nonce.
2412    pub(crate) fn swap_hello_admission(
2413        &self,
2414        module_id: &str,
2415        presented: Option<&str>,
2416    ) -> SwapHelloAdmission {
2417        let swaps = self
2418            .swaps
2419            .lock()
2420            .unwrap_or_else(|poisoned| poisoned.into_inner());
2421        let Some(swap) = swaps.get(module_id) else {
2422            return SwapHelloAdmission::NotSwapping;
2423        };
2424        let Some(presented) = presented else {
2425            return SwapHelloAdmission::Refused;
2426        };
2427        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428            return if swap.candidate_admitted {
2429                SwapHelloAdmission::Refused
2430            } else {
2431                SwapHelloAdmission::Candidate
2432            };
2433        }
2434        if swap
2435            .incumbent_nonce
2436            .as_deref()
2437            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438        {
2439            return SwapHelloAdmission::NotSwapping;
2440        }
2441        SwapHelloAdmission::Refused
2442    }
2443
2444    /// Record that the swap token has registered a candidate, so it admits no
2445    /// second HELLO.
2446    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447        if let Some(swap) = self
2448            .swaps
2449            .lock()
2450            .unwrap_or_else(|poisoned| poisoned.into_inner())
2451            .get_mut(module_id)
2452        {
2453            swap.candidate_admitted = true;
2454        }
2455    }
2456
2457    /// Test/support lookup for the current launch nonce of a supervised spawn.
2458    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459        self.spawn_nonces
2460            .lock()
2461            .unwrap_or_else(|poisoned| poisoned.into_inner())
2462            .get(module_id)
2463            .cloned()
2464    }
2465
2466    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2467    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468        self.reserved_nonces
2469            .lock()
2470            .unwrap_or_else(|poisoned| poisoned.into_inner())
2471            .get(module_id)
2472            .cloned()
2473            .flatten()
2474    }
2475
2476    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477        // Normally already marked before the process was spawned; marking here
2478        // too keeps `configured_ids` a superset of the roster for any caller
2479        // that inserts a module directly.
2480        self.mark_configured(module.module_id());
2481        let mut modules = self
2482            .modules
2483            .lock()
2484            .unwrap_or_else(|poisoned| poisoned.into_inner());
2485        modules.insert(module.module_id().to_string(), module)
2486    }
2487
2488    /// Record that the supervisor has taken on `module_id`. Called before the
2489    /// module's first process is spawned, so that by the time that process can
2490    /// register, [`Self::is_configured`] already answers true.
2491    fn mark_configured(&self, module_id: &str) {
2492        self.configured_ids
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner())
2495            .insert(module_id.to_string());
2496    }
2497
2498    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2499    /// before it was ever put on the roster. A module already on the roster
2500    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2501    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502        let modules = self
2503            .modules
2504            .lock()
2505            .unwrap_or_else(|poisoned| poisoned.into_inner());
2506        if !modules.contains_key(module_id) {
2507            self.configured_ids
2508                .lock()
2509                .unwrap_or_else(|poisoned| poisoned.into_inner())
2510                .remove(module_id);
2511        }
2512    }
2513
2514    /// Whether `module_id` is a module this daemon supervises: on the roster,
2515    /// or about to be (its process is being spawned right now).
2516    ///
2517    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2518    /// for scopes: a supervised module's process can register and sync before
2519    /// [`Self::get`] can return it, and in that window it is still a module
2520    /// that will sync, not one that never will.
2521    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522        self.configured_ids
2523            .lock()
2524            .unwrap_or_else(|poisoned| poisoned.into_inner())
2525            .contains(module_id)
2526    }
2527
2528    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529        let modules = self
2530            .modules
2531            .lock()
2532            .unwrap_or_else(|poisoned| poisoned.into_inner());
2533        modules.get(module_id).cloned()
2534    }
2535
2536    pub(crate) fn record_late_health_answer(
2537        &self,
2538        module_id: &str,
2539        latency_ms: u64,
2540    ) -> Result<bool, SuperviseError> {
2541        let Some(module) = self.get(module_id) else {
2542            return Ok(false);
2543        };
2544        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546            state.health.last_late_answer_latency_ms = Some(latency_ms);
2547            // A late answer is an answer: the module served the probe, just past
2548            // the deadline. Leaving the miss streak in place while logging
2549            // "proves the module is alive" is how a CPU-starved module that
2550            // answers every probe a few seconds late still marches to the
2551            // threshold and gets killed — the exact kill class `NoAnswer` is
2552            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2553            // is degradation, and degradation reports; it does not restart.
2554            state.health.consecutive_failures = 0;
2555        })?;
2556        Ok(true)
2557    }
2558
2559    /// Arm the one-shot marker for the module process that this caller
2560    /// deliberately initiated severance against. Generic connection teardown
2561    /// must not call this:
2562    /// a surviving process would otherwise retain an exemption for a later
2563    /// genuine crash.
2564    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565        let Some(module) = self.get(module_id) else {
2566            return Ok(false);
2567        };
2568        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570            return Ok(false);
2571        };
2572        drop(snapshot);
2573        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574    }
2575
2576    pub fn list(&self) -> Vec<SupervisedModule> {
2577        let modules = self
2578            .modules
2579            .lock()
2580            .unwrap_or_else(|poisoned| poisoned.into_inner());
2581        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583        modules
2584    }
2585
2586    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587        self.activity_images
2588            .lock()
2589            .unwrap_or_else(|p| p.into_inner())
2590            .remove(module_id);
2591        self.spawn_nonces
2592            .lock()
2593            .unwrap_or_else(|poisoned| poisoned.into_inner())
2594            .remove(module_id);
2595        self.close_swap(module_id);
2596        let mut reserved_nonces = self
2597            .reserved_nonces
2598            .lock()
2599            .unwrap_or_else(|poisoned| poisoned.into_inner());
2600        if reserved_nonces.contains_key(module_id) {
2601            // The old nonce must die with the removed process, but the exact-id
2602            // gate remains until an operator explicitly releases it.
2603            reserved_nonces.insert(module_id.to_string(), None);
2604        }
2605        drop(reserved_nonces);
2606        self.reserved_prefix_owners
2607            .lock()
2608            .unwrap_or_else(|poisoned| poisoned.into_inner())
2609            .retain(|_, owner| owner != module_id);
2610        let removed = self
2611            .modules
2612            .lock()
2613            .unwrap_or_else(|poisoned| poisoned.into_inner())
2614            .remove(module_id);
2615        self.configured_ids
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        removed
2620    }
2621
2622    /// Remember a module removed by a non-preview rescan so route.open can
2623    /// distinguish that intentional removal from an unknown id.
2624    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625        self.removal_tombstones
2626            .lock()
2627            .unwrap_or_else(|poisoned| poisoned.into_inner())
2628            .insert(module_id.to_string(), unix_ms_now());
2629    }
2630
2631    /// Return how long ago a rescan removed this module in milliseconds.
2632    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633        self.removal_tombstones
2634            .lock()
2635            .unwrap_or_else(|poisoned| poisoned.into_inner())
2636            .get(module_id)
2637            .copied()
2638            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639    }
2640
2641    /// Retire a reserved-id gate only after its module has left supervision.
2642    ///
2643    /// A retained gate has no live nonce (`None`), so releasing any other entry
2644    /// would weaken a currently configured or otherwise active reservation.
2645    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646        if self.get(module_id).is_some() {
2647            return false;
2648        }
2649        let mut reserved_nonces = self
2650            .reserved_nonces
2651            .lock()
2652            .unwrap_or_else(|poisoned| poisoned.into_inner());
2653        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654            return false;
2655        }
2656        reserved_nonces.remove(module_id);
2657        true
2658    }
2659
2660    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661        Arc::clone(&self.operation_lock)
2662    }
2663}
2664
2665/// Process supervisor for subc-owned singleton modules.
2666#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668    registry: Arc<Registry>,
2669    restart_policy: RestartPolicy,
2670    drain_timeout: Duration,
2671    connection_file_path: Option<PathBuf>,
2672    capture_logs_dir: Option<PathBuf>,
2673    forwarding: Option<Arc<ForwardingTable>>,
2674    process_liveness: Arc<SupervisorProcessLiveness>,
2675    supervisor_handle: Option<SupervisorHandle>,
2676    health: HealthConfig,
2677    daemon_start_clock: crate::clock::StartClock,
2678    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679    spawn_events: SpawnEventFeed,
2680    provenance_probe: ExecutableIdentityProbe,
2681    /// Every process spawned through this supervisor (and its clones) and not
2682    /// yet reaped, so daemon shutdown can end them.
2683    child_roster: ChildRoster,
2684    #[cfg(target_os = "linux")]
2685    cgroup_placement: Option<subc_cgroup::Placement>,
2686    #[cfg(test)]
2687    test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690/// A roster entry whose monitor and first process have not started yet.
2691/// Keeping the command receiver here lets discovery serve status immediately
2692/// while start/stop requests wait until bootstrap has published the connection file.
2693pub(crate) struct RegisteredConfiguredModule {
2694    spec: ModuleSpec,
2695    runtime: SupervisorRuntimeConfig,
2696    module: SupervisedModule,
2697    commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700/// Test-only hook run on the path that takes on a new module, right after its
2701/// first `spawn_child` returns (the process exists and could already be
2702/// registering) and before that process is handed to the module's supervise
2703/// loop and put on the roster. Lets a test observe what a fast child would see
2704/// in that window without racing a real one.
2705#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715        f.write_str("AfterFirstSpawnHook")
2716    }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721    fn run(&self, module_id: &str) {
2722        if let Some(hook) = &self.0 {
2723            hook(module_id);
2724        }
2725    }
2726}
2727
2728impl Supervisor {
2729    #[cfg(test)]
2730    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731        let supervisor = Self::new(registry, policy);
2732        #[cfg(target_os = "macos")]
2733        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734        supervisor
2735    }
2736    /// Verify the trampoline once when configured. Missing private OS support
2737    /// refuses every macOS launch by name but does not stop the daemon's control
2738    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2739    /// entry point; the library must not exec an arbitrary hosting program.
2740    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741        let path = path.into();
2742        #[cfg(target_os = "macos")]
2743        {
2744            let result = probe_privacy_trampoline(&path).map(|()| path);
2745            if let Err(cause) = &result {
2746                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747            }
2748            self.child_roster.set_privacy_trampoline(result);
2749        }
2750        #[cfg(not(target_os = "macos"))]
2751        let _ = path;
2752        self
2753    }
2754    /// The first step of an announced daemon shutdown, before the notice and
2755    /// before any connection is closed.
2756    ///
2757    /// Sets the daemon-shutdown flag first: from here on no module is
2758    /// respawned (crash restart, operator restart, or swap), and every child
2759    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2760    /// the module exits on the EOF this shutdown gives it or is signalled by a
2761    /// service manager that kills the whole cgroup. Then writes the journal's
2762    /// shutdown marker, which records the instant and closes this daemon
2763    /// incarnation's stretch of the journal.
2764    #[cfg(unix)]
2765    pub(crate) fn begin_daemon_shutdown(&self) {
2766        self.child_roster.close();
2767        if let Some(journal) = &self.terminal_journal {
2768            journal.stamp_shutdown();
2769        }
2770    }
2771
2772    /// Announce a cut while established connections can still carry replies.
2773    /// These budgets promise notice and a bounded wait, not child completion;
2774    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2775    #[cfg(unix)]
2776    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2777        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2778        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2779        let Some(forwarding) = &self.forwarding else {
2780            return Ok(());
2781        };
2782        let modules = forwarding
2783            .begin_daemon_drain()
2784            .map_err(SuperviseError::Forwarding)?;
2785        let deadline_ms =
2786            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2787        let mut notices = tokio::task::JoinSet::new();
2788        let mut drains = Vec::new();
2789        for (module_id, endpoint) in modules {
2790            let Some(target) = forwarding
2791                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2792                .map_err(SuperviseError::Forwarding)?
2793            else {
2794                continue;
2795            };
2796            let routes = forwarding
2797                .endpoint_routes(target.endpoint)
2798                .map_err(SuperviseError::Forwarding)?;
2799            // Restart allows deployed consumers to reopen after the new daemon
2800            // appears. The wire reason stays `restart`; what tells a daemon cut
2801            // apart from a module restart afterwards is the terminal record
2802            // itself, whose disposition is `daemon_shutdown` for every exit
2803            // observed once `begin_daemon_shutdown` has run.
2804            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2805                reason: RouteCloseReason::Restart,
2806                deadline_ms,
2807            })
2808            .expect("module draining serializes");
2809            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2810            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2811            for route in routes {
2812                let client = route.goodbye_target;
2813                if let Some((_, channels)) = clients
2814                    .iter_mut()
2815                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2816                {
2817                    channels.push(client.channel);
2818                } else {
2819                    let channel = client.channel;
2820                    clients.push((client, vec![channel]));
2821                }
2822            }
2823            for (client, mut channels) in clients {
2824                channels.sort_unstable();
2825                channels.dedup();
2826                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2827                    module_id: module_id.clone(),
2828                    channels,
2829                    reason: RouteCloseReason::Restart,
2830                })
2831                .expect("route closing serializes");
2832                recipients.push((client.sink, client.negotiated_ver, closing));
2833            }
2834            for (sink, version, body) in recipients {
2835                notices.spawn(async move {
2836                    let frame = Frame::build_with_version(
2837                        version,
2838                        FrameType::Push,
2839                        control_flags(),
2840                        0,
2841                        0,
2842                        0,
2843                        body,
2844                    )
2845                    .expect("bounded lifecycle notice frame builds");
2846                    sink.send_flushed(frame).await
2847                });
2848            }
2849            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2850            drains.push((module_id, target.endpoint, gauges));
2851        }
2852        // A quiet forwarding table is not proof that queued notices reached the
2853        // socket. Wait for writer flush acknowledgements before testing quiescence.
2854        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2855        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2856            if !matches!(result, Ok(Ok(()))) {
2857                warn!(?result, "daemon shutdown notice delivery failed");
2858            }
2859        }
2860        notices.abort_all();
2861        let deadline = Instant::now() + DRAIN_BUDGET;
2862        let mut waits = tokio::task::JoinSet::new();
2863        for (module_id, endpoint, gauges) in drains {
2864            let forwarding = Arc::clone(forwarding);
2865            let mut runtime = self.runtime_config();
2866            runtime.health.cadence = Duration::from_millis(100);
2867            waits.spawn(async move {
2868                let wait_result = wait_for_forwarding_quiescence(
2869                    &forwarding,
2870                    &module_id,
2871                    &runtime,
2872                    endpoint,
2873                    deadline,
2874                    &gauges,
2875                    DrainScope::Endpoint(endpoint),
2876                    &mut DrainGaugeTrace::new(&gauges),
2877                )
2878                .await;
2879                // Store the measurement before the task returns: quiescence
2880                // may be reached just as the deadline stops the join loop from
2881                // collecting results. Unfinished or failed waits report false.
2882                forwarding
2883                    .record_daemon_drain_result(
2884                        endpoint,
2885                        drained_after_quiescence_wait(&wait_result),
2886                    )
2887                    .map_err(SuperviseError::Forwarding)?;
2888                wait_result
2889            });
2890        }
2891        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2892            if !matches!(result, Ok(Ok(true))) {
2893                warn!(?result, "daemon shutdown drain did not reach quiescence");
2894            }
2895        }
2896        Ok(())
2897    }
2898
2899    /// The last step of an announced daemon shutdown, after the notice and the
2900    /// drain: send every registered module a module GOODBYE, the same planned
2901    /// stop signal `ck module stop` gives, then close every connection so each
2902    /// subc module sees EOF and starts its own teardown, then end every
2903    /// supervised child that has not exited
2904    /// by its own deadline (its drain budget, capped). Modules lead their own
2905    /// process groups, so a
2906    /// service manager's group kill no longer reaches them; without this a
2907    /// child that does not stop on EOF (every `protocol: "none"` child, which
2908    /// has no connection) would outlive the daemon. Every wait is bounded (see
2909    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2910    #[cfg(unix)]
2911    pub(crate) async fn end_children_for_daemon_shutdown(
2912        &self,
2913        already_escalated: bool,
2914        escalate: impl std::future::Future<Output = ()>,
2915    ) {
2916        tokio::pin!(escalate);
2917        let mut escalated = already_escalated;
2918        if let Some(forwarding) = &self.forwarding {
2919            let reason = CloseReason::new(
2920                "daemon_shutdown",
2921                "the daemon is exiting after its shutdown notice and drain",
2922            );
2923            if escalated {
2924                // The operator asked to stop waiting: queue the GOODBYEs but
2925                // do not wait for them to be written.
2926                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2927            } else {
2928                tokio::select! {
2929                    biased;
2930                    _ = escalate.as_mut() => {
2931                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2932                        escalated = true;
2933                    }
2934                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2935                }
2936            }
2937            let closed = forwarding.close_all_connections(&reason);
2938            debug!(closed, "closed established connections for daemon shutdown");
2939        }
2940        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2941        // already completed and must not be polled again; the child shutdown
2942        // wait is told it is escalated and gets a future that never fires.
2943        let escalated_here = escalated && !already_escalated;
2944        let remaining_escalate = async move {
2945            if escalated_here {
2946                std::future::pending::<()>().await;
2947            } else {
2948                escalate.await;
2949            }
2950        };
2951        crate::child_roster::end_children_for_daemon_shutdown(
2952            &self.child_roster,
2953            escalated,
2954            remaining_escalate,
2955        )
2956        .await;
2957    }
2958
2959    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2960        Self {
2961            registry,
2962            restart_policy,
2963            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2964            connection_file_path: None,
2965            capture_logs_dir: None,
2966            forwarding: None,
2967            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2968            supervisor_handle: None,
2969            health: HealthConfig::default(),
2970            daemon_start_clock: crate::clock::StartClock::capture(),
2971            terminal_journal: None,
2972            spawn_events: SpawnEventFeed::default(),
2973            provenance_probe: ExecutableIdentityProbe::default(),
2974            child_roster: ChildRoster::default(),
2975            #[cfg(target_os = "linux")]
2976            cgroup_placement: None,
2977            #[cfg(test)]
2978            test_after_first_spawn: AfterFirstSpawnHook::default(),
2979        }
2980    }
2981
2982    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2983        self.drain_timeout = drain_timeout;
2984        self
2985    }
2986
2987    pub fn with_process_liveness(
2988        mut self,
2989        process_liveness: Arc<SupervisorProcessLiveness>,
2990    ) -> Self {
2991        self.process_liveness = process_liveness;
2992        self
2993    }
2994
2995    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2996        self.connection_file_path = Some(connection_file_path.into());
2997        self
2998    }
2999
3000    /// Enables daemon-owned capture files for supervised stdout and stderr.
3001    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3002        self.capture_logs_dir = Some(logs_dir.into());
3003        self
3004    }
3005
3006    /// Names this daemon lifetime in spawn events, independently of whether a
3007    /// terminal journal is configured.
3008    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3009        // A millisecond start stamp can repeat after clock rollback or a rapid
3010        // restart. Use the connection file's random daemon_id instead: it already
3011        // identifies this daemon lifetime independently of the wall clock.
3012        self.spawn_events.configure_incarnation(daemon_incarnation);
3013        self
3014    }
3015
3016    /// Enables best-effort history shared by every supervised module. Without
3017    /// it, terminal history is kept only in each module's in-memory ring.
3018    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3019        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3020        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3021            path,
3022            daemon_incarnation,
3023        )));
3024        this
3025    }
3026
3027    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3028        self.forwarding = Some(forwarding);
3029        self
3030    }
3031
3032    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3033        self.spawn_events = supervisor_handle.spawn_events.clone();
3034        self.supervisor_handle = Some(supervisor_handle);
3035        self
3036    }
3037
3038    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3039        self.health = health;
3040        self
3041    }
3042
3043    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3044    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3045    /// record is kept.
3046    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3047        self.child_roster.record_to(path.into());
3048        self
3049    }
3050
3051    #[cfg(target_os = "linux")]
3052    pub fn with_cgroup_placement(
3053        mut self,
3054        cgroup_placement: Option<subc_cgroup::Placement>,
3055    ) -> Self {
3056        self.cgroup_placement = cgroup_placement;
3057        self
3058    }
3059
3060    /// Spawn `spec.program` and start monitoring it.
3061    ///
3062    /// The child is expected to parse `--subc <connection-file-path>`, read the
3063    /// TCP+key connection file, authenticate to the already-running listener, and
3064    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3065    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3066        validate_spec(&spec)?;
3067        self.establish_identity(&spec);
3068
3069        let runtime = self.runtime_config();
3070        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3071        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3072        let spawned = spawn_child(
3073            &spec,
3074            runtime.connection_file_path.as_deref(),
3075            self.supervisor_handle.as_ref(),
3076            &runtime.stderr_ring,
3077            runtime.capture_logs_dir.as_deref(),
3078            &runtime.child_roster,
3079            #[cfg(target_os = "linux")]
3080            runtime.cgroup_placement.as_ref(),
3081        );
3082        #[cfg(test)]
3083        self.test_after_first_spawn.run(&spec.module_id);
3084        let child = match spawned {
3085            Ok(child) => child,
3086            Err(err) => {
3087                // Unlike the configured paths, a failed `spawn` leaves nothing
3088                // on the roster, so the module must not stay marked configured.
3089                self.abandon_unrostered(&spec.module_id);
3090                return Err(err);
3091            }
3092        };
3093        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3094        if let Some(spawn) = activity_spawn {
3095            spawn.complete(&snapshot);
3096        }
3097
3098        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3099    }
3100
3101    /// Make `spec`'s module count as configured, with its identity gates
3102    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3103    /// exists.
3104    ///
3105    /// Every path that takes on a new module calls this before `spawn_child`.
3106    /// The order is the point: the child can connect, register, sync its
3107    /// scopes and ask about them as soon as it is spawned, and the module is
3108    /// only put on the roster after `spawn_child` returns. Were the mark set
3109    /// with the roster entry, a fast child would see its own owner reported
3110    /// as not configured, and a scoped `route.open` in that window would be
3111    /// refused as terminal `scope_not_live` ("will never sync") instead of
3112    /// retryable `scope_not_synced`.
3113    fn establish_identity(&self, spec: &ModuleSpec) {
3114        if let Some(supervisor_handle) = &self.supervisor_handle {
3115            supervisor_handle.apply_identity_configuration(spec);
3116            supervisor_handle.mark_configured(&spec.module_id);
3117        }
3118    }
3119
3120    /// Take back [`Self::establish_identity`]'s configured mark when the
3121    /// module will not be put on the roster after all.
3122    fn abandon_unrostered(&self, module_id: &str) {
3123        if let Some(supervisor_handle) = &self.supervisor_handle {
3124            supervisor_handle.unmark_configured_unless_rostered(module_id);
3125        }
3126    }
3127
3128    /// Record a freshly spawned first process as running. On failure the
3129    /// module never reaches the roster, so its configured mark is taken back.
3130    fn mark_first_process_running(
3131        &self,
3132        spec: &ModuleSpec,
3133        runtime: &SupervisorRuntimeConfig,
3134        snapshot: &SharedSnapshot,
3135        child: &SupervisedChild,
3136    ) -> Result<(), SuperviseError> {
3137        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3138            self.abandon_unrostered(&spec.module_id);
3139            return Err(err);
3140        }
3141        self.process_liveness
3142            .track(spec.module_id.clone(), Arc::clone(snapshot));
3143        Ok(())
3144    }
3145
3146    /// Start supervising a module declared in daemon configuration.
3147    ///
3148    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3149    /// failures in the supervisor handle so operator-facing `supervisor.list`
3150    /// reflects every configured module while daemon startup continues.
3151    pub fn supervise_configured(
3152        &self,
3153        spec: ModuleSpec,
3154        enabled: bool,
3155    ) -> Result<SupervisedModule, SuperviseError> {
3156        self.supervise_configured_requested(
3157            spec,
3158            enabled,
3159            self.health.clone(),
3160            None,
3161            self.restart_policy,
3162            None,
3163        )
3164    }
3165
3166    /// Supervise a configured module with its own health, drain, and crash
3167    /// budget. The restart policy is per-module because the config file is:
3168    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3169    /// module that is expensive to restart should not be forced onto the same
3170    /// budget as one that is cheap.
3171    pub fn supervise_configured_with_health(
3172        &self,
3173        spec: ModuleSpec,
3174        enabled: bool,
3175        health: HealthConfig,
3176        drain_timeout_ms: Option<u64>,
3177        restart_policy: RestartPolicy,
3178    ) -> Result<SupervisedModule, SuperviseError> {
3179        self.supervise_configured_requested(
3180            spec,
3181            enabled,
3182            health,
3183            drain_timeout_ms,
3184            restart_policy,
3185            None,
3186        )
3187    }
3188
3189    pub(crate) fn supervise_configured_requested(
3190        &self,
3191        spec: ModuleSpec,
3192        enabled: bool,
3193        health: HealthConfig,
3194        drain_timeout_ms: Option<u64>,
3195        restart_policy: RestartPolicy,
3196        requester: Option<Principal>,
3197    ) -> Result<SupervisedModule, SuperviseError> {
3198        let registered = self.register_configured_with_health(
3199            spec,
3200            enabled,
3201            health,
3202            drain_timeout_ms,
3203            restart_policy,
3204        )?;
3205        self.start_registered_configured(registered, requester)
3206    }
3207
3208    /// Record a configured module as starting or disabled, without launching a
3209    /// process or running its monitor. Bootstrap registers the entire roster
3210    /// before publishing discovery, then activates it once children can read
3211    /// the connection file passed in their `--subc` argument.
3212    pub(crate) fn register_configured_with_health(
3213        &self,
3214        spec: ModuleSpec,
3215        enabled: bool,
3216        health: HealthConfig,
3217        drain_timeout_ms: Option<u64>,
3218        restart_policy: RestartPolicy,
3219    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3220        validate_spec(&spec)?;
3221        self.establish_identity(&spec);
3222
3223        let mut runtime = self.runtime_config();
3224        runtime.health = health;
3225        runtime.restart_policy = restart_policy;
3226        if let Some(ms) = drain_timeout_ms {
3227            runtime.drain_timeout = Duration::from_millis(ms);
3228            *runtime
3229                .effective_drain_timeout
3230                .lock()
3231                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3232        }
3233        let snapshot = if enabled {
3234            SupervisorSnapshot::starting()
3235        } else {
3236            SupervisorSnapshot::disabled()
3237        };
3238        let snapshot = {
3239            let mut snapshot = snapshot;
3240            if enabled {
3241                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3242                {
3243                    snapshot.waiting_for_previous_pid = Some(pid);
3244                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3245                }
3246            }
3247            snapshot
3248        };
3249        let (module, commands) =
3250            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3251        Ok(RegisteredConfiguredModule {
3252            spec,
3253            runtime,
3254            module,
3255            commands,
3256        })
3257    }
3258
3259    /// Activate a previously registered module. Queued control requests are
3260    /// served by the monitor after the initial configured spawn has settled.
3261    pub(crate) fn start_registered_configured(
3262        &self,
3263        registered: RegisteredConfiguredModule,
3264        requester: Option<Principal>,
3265    ) -> Result<SupervisedModule, SuperviseError> {
3266        let RegisteredConfiguredModule {
3267            spec,
3268            mut runtime,
3269            module,
3270            commands,
3271        } = registered;
3272        runtime.start_requester = requester;
3273        let snapshot = &module.inner.snapshot;
3274        let should_spawn = {
3275            let state = lock_snapshot(snapshot)?;
3276            state.enabled && state.waiting_for_previous_pid.is_none()
3277        };
3278        let child = if should_spawn {
3279            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3280            let spawned = spawn_child(
3281                &spec,
3282                runtime.connection_file_path.as_deref(),
3283                self.supervisor_handle.as_ref(),
3284                &runtime.stderr_ring,
3285                runtime.capture_logs_dir.as_deref(),
3286                &runtime.child_roster,
3287                #[cfg(target_os = "linux")]
3288                runtime.cgroup_placement.as_ref(),
3289            );
3290            #[cfg(test)]
3291            self.test_after_first_spawn.run(&spec.module_id);
3292            match spawned {
3293                Ok(child) => {
3294                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3295                    if let Some(spawn) = activity_spawn {
3296                        spawn.complete(snapshot);
3297                    }
3298                    Some(child)
3299                }
3300                Err(err) => {
3301                    if runtime.health.critical {
3302                        error!(
3303                            module_id = %spec.module_id,
3304                            program = %spec.program.display(),
3305                            error = %err,
3306                            "critical configured module failed to spawn; marking failed and alerting"
3307                        );
3308                    } else {
3309                        error!(
3310                            module_id = %spec.module_id,
3311                            program = %spec.program.display(),
3312                            error = %err,
3313                            "configured module failed to spawn; marking failed and continuing"
3314                        );
3315                    }
3316                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3317                        state.state = ModuleState::Failed;
3318                        clear_current_process_facts(state);
3319                    })?;
3320                    None
3321                }
3322            }
3323        } else {
3324            None
3325        };
3326        self.start_monitor(spec, runtime, &module, child, commands);
3327        Ok(module)
3328    }
3329
3330    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3331        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3332        SupervisorRuntimeConfig {
3333            start_requester: None,
3334            pending_activity_restart: false,
3335            provenance_probe: self.provenance_probe.clone(),
3336            scheduled_respawn: Arc::default(),
3337            deferred_reload_reply: Arc::default(),
3338            restart_policy: self.restart_policy,
3339            drain_timeout: self.drain_timeout,
3340            // Shared with this module's roster copy: daemon shutdown waits on
3341            // each child for the module's own drain budget, as resolved now.
3342            child_roster: self
3343                .child_roster
3344                .for_module(Arc::clone(&effective_drain_timeout)),
3345            effective_drain_timeout,
3346            default_drain_timeout: self.drain_timeout,
3347            health: self.health.clone(),
3348            connection_file_path: self.connection_file_path.clone(),
3349            capture_logs_dir: self.capture_logs_dir.clone(),
3350            forwarding: self.forwarding.clone(),
3351            supervisor_handle: self.supervisor_handle.clone(),
3352            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3353            terminal_ring: Arc::new(Mutex::new(
3354                TerminalRing::new(
3355                    TerminalRingConfig::default(),
3356                    self.daemon_start_clock.started_at_ms(),
3357                )
3358                .with_start_clock(self.daemon_start_clock)
3359                .with_journal(self.terminal_journal.clone())
3360                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3361            )),
3362            spawn_events: self.spawn_events.clone(),
3363            #[cfg(target_os = "linux")]
3364            cgroup_placement: self.cgroup_placement.clone(),
3365            #[cfg(test)]
3366            test_seed_stale_facts_before_enable_spawn: false,
3367            #[cfg(test)]
3368            test_reload_exit_record_gate: None,
3369        }
3370    }
3371
3372    fn supervised_module(
3373        &self,
3374        spec: ModuleSpec,
3375        runtime: SupervisorRuntimeConfig,
3376        snapshot: SharedSnapshot,
3377        child: Option<SupervisedChild>,
3378    ) -> SupervisedModule {
3379        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3380        self.start_monitor(spec, runtime, &module, child, commands);
3381        module
3382    }
3383
3384    fn register_module(
3385        &self,
3386        spec: &ModuleSpec,
3387        runtime: &SupervisorRuntimeConfig,
3388        snapshot: SharedSnapshot,
3389    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3390        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3391            spec: spec.clone(),
3392            health: runtime.health.clone(),
3393        }));
3394        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3395        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3396        // The module's OWN policy, which may be its per-module config rather than
3397        // the supervisor-wide one; status must report the budget the supervise
3398        // loop actually enforces.
3399        let restart_policy = runtime.restart_policy;
3400        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3401        let (tx, rx) = mpsc::channel(4);
3402        let module_id = spec.module_id.clone();
3403        let module = SupervisedModule {
3404            inner: Arc::new(SupervisedModuleInner {
3405                module_id: module_id.clone(),
3406                registry: Arc::clone(&self.registry),
3407                snapshot,
3408                configuration,
3409                stderr_ring,
3410                terminal_ring,
3411                commands: tx,
3412                monitor: Mutex::new(None),
3413                restart_policy,
3414                effective_drain_timeout,
3415                provenance_probe: self.provenance_probe.clone(),
3416            }),
3417        };
3418        // Identity gates and the configured mark must precede the roster entry,
3419        // so a client never sees a module without its admission policy.
3420        if let Some(supervisor_handle) = &self.supervisor_handle {
3421            supervisor_handle.insert(module.clone());
3422        }
3423        (module, rx)
3424    }
3425
3426    fn start_monitor(
3427        &self,
3428        spec: ModuleSpec,
3429        runtime: SupervisorRuntimeConfig,
3430        module: &SupervisedModule,
3431        child: Option<SupervisedChild>,
3432        commands: mpsc::Receiver<SupervisorCommand>,
3433    ) {
3434        let monitor = tokio::spawn(supervise_loop(
3435            spec,
3436            runtime,
3437            Arc::clone(&self.registry),
3438            Arc::clone(&self.process_liveness),
3439            Arc::clone(&module.inner.snapshot),
3440            child,
3441            commands,
3442        ));
3443        *module
3444            .inner
3445            .monitor
3446            .lock()
3447            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3448    }
3449}
3450
3451impl Default for Supervisor {
3452    fn default() -> Self {
3453        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3454    }
3455}
3456
3457/// Handle to one supervised child process.
3458#[derive(Clone)]
3459pub struct SupervisedModule {
3460    inner: Arc<SupervisedModuleInner>,
3461}
3462
3463struct SupervisedModuleInner {
3464    module_id: String,
3465    registry: Arc<Registry>,
3466    snapshot: SharedSnapshot,
3467    configuration: Arc<Mutex<SupervisedConfiguration>>,
3468    stderr_ring: Arc<Mutex<StderrRing>>,
3469    terminal_ring: Arc<Mutex<TerminalRing>>,
3470    commands: mpsc::Sender<SupervisorCommand>,
3471    monitor: Mutex<Option<JoinHandle<()>>>,
3472    /// Copied from the supervisor's runtime config at spawn so `status()` can
3473    /// report the restart budget without reaching back into the supervisor. The
3474    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3475    restart_policy: RestartPolicy,
3476    effective_drain_timeout: Arc<Mutex<Duration>>,
3477    provenance_probe: ExecutableIdentityProbe,
3478}
3479
3480impl fmt::Debug for SupervisedModule {
3481    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3482        f.debug_struct("SupervisedModule")
3483            .field("module_id", &self.inner.module_id)
3484            .field("status", &self.status())
3485            .finish_non_exhaustive()
3486    }
3487}
3488
3489impl SupervisedModule {
3490    pub fn module_id(&self) -> &str {
3491        &self.inner.module_id
3492    }
3493
3494    /// Test-only: put one probe miss on the streak, the way
3495    /// `handle_health_probe_failure` does, so tests can assert what a later
3496    /// event does to the streak without driving the whole probe loop.
3497    #[cfg(test)]
3498    pub(crate) fn record_health_probe_failure_for_test(
3499        &self,
3500        detail: &str,
3501    ) -> Result<(), SuperviseError> {
3502        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3503            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3504            state.health.detail = Some(detail.to_string());
3505        })
3506    }
3507
3508    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3509        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3510    }
3511
3512    pub(crate) fn registration_end_reason(
3513        &self,
3514    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3515        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3516        Ok(match snapshot.state {
3517            ModuleState::Draining if snapshot.draining_to_replace => {
3518                Some(RegistrationEndReason::SupervisorRestart)
3519            }
3520            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3521            ModuleState::Restarting if snapshot.draining_to_replace => {
3522                Some(RegistrationEndReason::SupervisorRestart)
3523            }
3524            _ => None,
3525        })
3526    }
3527
3528    /// The module's retained stderr, newest lines last.
3529    ///
3530    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3531    /// module, `supervisor.list` renders every module, and putting it in the
3532    /// shared snapshot would make each status read carry a payload almost nobody
3533    /// asked for. Callers that want the text ask for it.
3534    pub fn stderr_tail(
3535        &self,
3536        max_lines: Option<usize>,
3537        max_bytes: Option<usize>,
3538    ) -> StderrTailSnapshot {
3539        self.inner
3540            .stderr_ring
3541            .lock()
3542            .unwrap_or_else(|poisoned| poisoned.into_inner())
3543            .snapshot(max_lines, max_bytes)
3544    }
3545
3546    /// The module's bounded terminal history, oldest retained exit first.
3547    ///
3548    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3549    /// daemon whose in-memory history was necessarily reset.
3550    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3551        self.inner
3552            .terminal_ring
3553            .lock()
3554            .unwrap_or_else(|poisoned| poisoned.into_inner())
3555            .snapshot()
3556    }
3557
3558    /// Retained observations from the current ring and all journal generations.
3559    ///
3560    /// Blocking: this reads the journal files. Async callers use
3561    /// [`Self::read_durable_terminal_history`].
3562    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3563        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3564    }
3565
3566    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3567    /// read (up to every retained generation) never occupies a runtime worker.
3568    /// Fails only if the blocking task could not finish (runtime shutdown or a
3569    /// panic in the read).
3570    pub(crate) async fn read_durable_terminal_history(
3571        &self,
3572    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3573        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3574        let module_id = self.inner.module_id.clone();
3575        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3576            .await
3577    }
3578
3579    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3580        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3581            .map(|(status, _)| status)
3582    }
3583
3584    pub(crate) fn record_deliberate_severance(
3585        &self,
3586        identity: ProcessIdentity,
3587    ) -> Result<bool, SuperviseError> {
3588        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3589        if snapshot.pid != Some(identity.pid)
3590            || snapshot.process_start_time != Some(identity.start_time)
3591        {
3592            return Ok(false);
3593        }
3594        snapshot.deliberate_severance = Some(identity);
3595        Ok(true)
3596    }
3597
3598    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3599    ///
3600    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3601    /// does not produce reader-observability logs.
3602    pub(crate) fn status_for_control(
3603        &self,
3604        caller: &'static str,
3605    ) -> Result<ModuleStatus, SuperviseError> {
3606        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3607            .map(|(status, _)| status)
3608    }
3609
3610    fn status_with_snapshot_lock(
3611        &self,
3612        snapshot: &SharedSnapshot,
3613        caller: Option<&'static str>,
3614    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3615        let mut guard = match caller {
3616            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3617            None => lock_snapshot(snapshot)?,
3618        };
3619        // Read the budget through the pruning path so a reader sees the same
3620        // in-window count the restart decision would use, not a stale total.
3621        let restart_count =
3622            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3623        let snapshot = guard.clone();
3624        drop(guard);
3625        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3626            SuperviseError::StatePoisoned {
3627                module_id: Some(self.inner.module_id.clone()),
3628            }
3629        })?;
3630        let registration_active = self
3631            .inner
3632            .registry
3633            .get_module(&self.inner.module_id)
3634            .map_err(SuperviseError::Registry)?
3635            .is_some();
3636        let protocol = snapshot
3637            .spawned_protocol
3638            .unwrap_or(self.declared_protocol()?);
3639        let running_process =
3640            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3641        // Registration is the difference between the two protocols and the only
3642        // one: a subc module that has not registered cannot serve a request even
3643        // though its process is up, and a `none` module never registers at all,
3644        // so requiring it there would pin `live` to false for the whole life of
3645        // a perfectly healthy process.
3646        let live = match protocol {
3647            ModuleProtocol::Subc => running_process && registration_active,
3648            ModuleProtocol::None => running_process,
3649        };
3650
3651        Ok((
3652            ModuleStatus {
3653                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3654                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3655                module_id: self.inner.module_id.clone(),
3656                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3657                state: snapshot.state,
3658                enabled: snapshot.enabled,
3659                process_alive: snapshot.process_alive,
3660                registration_active,
3661                protocol,
3662                live,
3663                restart_count,
3664                lifetime_restarts: snapshot.lifetime_restarts,
3665                spawn_generation: snapshot.spawn_generation,
3666                max_restarts: self.inner.restart_policy.max_restarts,
3667                restart_window: self.inner.restart_policy.window,
3668                drain_timeout,
3669                restart_backoff: self.inner.restart_policy.backoff,
3670                restart_max_backoff: self.inner.restart_policy.max_backoff,
3671                pid: snapshot.reported_pid(),
3672                spawned_at_ms: snapshot.spawned_at_ms,
3673                spawned_from: snapshot.spawned_from,
3674                launch_spec_digest: snapshot.launch_spec_digest,
3675                process_start_time: snapshot.process_start_time,
3676                last_exit: snapshot.last_exit,
3677                health: snapshot.health,
3678            },
3679            snapshot.spawned_file_identity.clone(),
3680        ))
3681    }
3682
3683    #[cfg(test)]
3684    pub(crate) fn hold_snapshot_for_test(
3685        &self,
3686        acquired: std::sync::mpsc::Sender<()>,
3687        hold: Duration,
3688    ) -> std::thread::JoinHandle<()> {
3689        let snapshot = Arc::clone(&self.inner.snapshot);
3690        std::thread::spawn(move || {
3691            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3692            acquired
3693                .send(())
3694                .expect("test receiver waits for snapshot lock");
3695            std::thread::sleep(hold);
3696        })
3697    }
3698
3699    /// The status and the running-image check for `supervisor.provenance`,
3700    /// taken from one status read. The exec acknowledgement can land between
3701    /// two separate reads, and the reply would then pair "no pid yet" with an
3702    /// image observed after the module started, which describes no single
3703    /// moment.
3704    pub(crate) async fn status_and_running_image_agreement(
3705        &self,
3706    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3707        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3708        let image = self
3709            .inner
3710            .provenance_probe
3711            .observe(
3712                status.pid,
3713                status.spawned_from.as_deref(),
3714                identity,
3715                status.process_start_time,
3716            )
3717            .await;
3718        Ok((status, image))
3719    }
3720
3721    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3722        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3723            Ok(snapshot) => snapshot.clone(),
3724            Err(_) => {
3725                return subc_control::RunningImageAgreement::Unavailable {
3726                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3727                };
3728            }
3729        };
3730        self.inner
3731            .provenance_probe
3732            .observe(
3733                snapshot.reported_pid(),
3734                snapshot.spawned_from.as_deref(),
3735                snapshot.spawned_file_identity.clone(),
3736                snapshot.process_start_time,
3737            )
3738            .await
3739    }
3740
3741    /// Memory and CPU time of the module's current process, read now. Only the
3742    /// process the supervisor spawned is read, not processes it has started.
3743    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3744        #[cfg(windows)]
3745        {
3746            let image = lock_snapshot(&self.inner.snapshot)
3747                .ok()
3748                .and_then(|snapshot| {
3749                    snapshot
3750                        .spawned_file_identity
3751                        .as_ref()
3752                        .and_then(|identity| identity.windows_image.clone())
3753                });
3754            match image {
3755                Some(image) => {
3756                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3757                }
3758                None => subc_control::ChildResourceUsage::Unavailable {
3759                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3760                },
3761            }
3762        }
3763        #[cfg(not(windows))]
3764        {
3765            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3766                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3767                Err(_) => {
3768                    return subc_control::ChildResourceUsage::Unavailable {
3769                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3770                    }
3771                }
3772            };
3773            crate::child_resources::read(pid, start_time)
3774        }
3775    }
3776
3777    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3778        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3779        Ok(match snapshot.state {
3780            ModuleState::Restarting => true,
3781            ModuleState::Failed | ModuleState::Disabled => false,
3782            _ if snapshot.enabled && snapshot.respawn_pending => true,
3783            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3784        })
3785    }
3786
3787    #[cfg(test)]
3788    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3789        self.is_warming_with_snapshot_lock(None)
3790    }
3791
3792    pub(crate) fn is_warming_for_control(
3793        &self,
3794        caller: &'static str,
3795    ) -> Result<bool, SuperviseError> {
3796        self.is_warming_with_snapshot_lock(Some(caller))
3797    }
3798
3799    fn is_warming_with_snapshot_lock(
3800        &self,
3801        caller: Option<&'static str>,
3802    ) -> Result<bool, SuperviseError> {
3803        let snapshot = match caller {
3804            Some(caller) => {
3805                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3806            }
3807            None => lock_snapshot(&self.inner.snapshot)?,
3808        }
3809        .clone();
3810        Ok(matches!(
3811            snapshot.state,
3812            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3813        ))
3814    }
3815
3816    /// Drain the module and stop monitoring it.
3817    pub async fn drain(&self) -> Result<(), SuperviseError> {
3818        self.stop().await
3819    }
3820
3821    pub(crate) async fn retire_requested(
3822        &self,
3823        requester: Option<Principal>,
3824    ) -> Result<(), SuperviseError> {
3825        match self.state()? {
3826            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3827            ModuleState::Starting
3828            | ModuleState::Running
3829            | ModuleState::Unresponsive
3830            | ModuleState::Restarting
3831            | ModuleState::Draining
3832            | ModuleState::Disabled => {}
3833        }
3834
3835        let (reply_tx, reply_rx) = oneshot::channel();
3836        self.inner
3837            .commands
3838            .send(SupervisorCommand::Retire {
3839                requester,
3840                reply: reply_tx,
3841            })
3842            .await
3843            .map_err(|_| SuperviseError::CommandClosed {
3844                module_id: self.inner.module_id.clone(),
3845            })?;
3846        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3847            module_id: self.inner.module_id.clone(),
3848        })?
3849    }
3850
3851    pub async fn stop(&self) -> Result<(), SuperviseError> {
3852        self.stop_requested(None).await
3853    }
3854
3855    pub(crate) async fn stop_requested(
3856        &self,
3857        requester: Option<Principal>,
3858    ) -> Result<(), SuperviseError> {
3859        match self.state()? {
3860            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3861            ModuleState::Starting
3862            | ModuleState::Running
3863            | ModuleState::Unresponsive
3864            | ModuleState::Restarting
3865            | ModuleState::Draining
3866            | ModuleState::Disabled => {}
3867        }
3868
3869        let (reply_tx, reply_rx) = oneshot::channel();
3870        self.inner
3871            .commands
3872            .send(SupervisorCommand::Drain {
3873                requester,
3874                reply: reply_tx,
3875            })
3876            .await
3877            .map_err(|_| SuperviseError::CommandClosed {
3878                module_id: self.inner.module_id.clone(),
3879            })?;
3880        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3881            module_id: self.inner.module_id.clone(),
3882        })?
3883    }
3884
3885    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3886        self.restart_requested(drain_timeout_ms, None).await
3887    }
3888
3889    pub(crate) async fn restart_requested(
3890        &self,
3891        drain_timeout_ms: Option<u64>,
3892        requester: Option<Principal>,
3893    ) -> Result<(), SuperviseError> {
3894        let queued_at = Instant::now();
3895        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3896        let (reply_tx, reply_rx) = oneshot::channel();
3897        self.inner
3898            .commands
3899            .send(SupervisorCommand::Restart {
3900                requester,
3901                drain_timeout_ms,
3902                received_at_generation,
3903                queued_at,
3904                reply: reply_tx,
3905            })
3906            .await
3907            .map_err(|_| SuperviseError::CommandClosed {
3908                module_id: self.inner.module_id.clone(),
3909            })?;
3910        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3911            module_id: self.inner.module_id.clone(),
3912        })?
3913    }
3914
3915    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3916    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3917    /// process then drains in the background of the supervise loop) or has
3918    /// failed, leaving the old process serving.
3919    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3920        self.swap_requested(ready_timeout, None).await
3921    }
3922
3923    pub(crate) async fn swap_requested(
3924        &self,
3925        ready_timeout: Option<Duration>,
3926        requester: Option<Principal>,
3927    ) -> Result<(), SuperviseError> {
3928        let (reply_tx, reply_rx) = oneshot::channel();
3929        self.inner
3930            .commands
3931            .send(SupervisorCommand::Swap {
3932                requester,
3933                ready_timeout,
3934                reply: reply_tx,
3935            })
3936            .await
3937            .map_err(|_| SuperviseError::CommandClosed {
3938                module_id: self.inner.module_id.clone(),
3939            })?;
3940        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3941            module_id: self.inner.module_id.clone(),
3942        })?
3943    }
3944
3945    pub async fn reload(&self) -> Result<(), SuperviseError> {
3946        self.reload_requested(None).await
3947    }
3948
3949    pub(crate) async fn reload_requested(
3950        &self,
3951        requester: Option<Principal>,
3952    ) -> Result<(), SuperviseError> {
3953        let (reply_tx, reply_rx) = oneshot::channel();
3954        self.inner
3955            .commands
3956            .send(SupervisorCommand::Reload {
3957                requester,
3958                reply: reply_tx,
3959            })
3960            .await
3961            .map_err(|_| SuperviseError::CommandClosed {
3962                module_id: self.inner.module_id.clone(),
3963            })?;
3964        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3965            module_id: self.inner.module_id.clone(),
3966        })?
3967    }
3968
3969    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3970        self.set_enabled_requested(enabled, None).await
3971    }
3972
3973    pub(crate) async fn set_enabled_requested(
3974        &self,
3975        enabled: bool,
3976        requester: Option<Principal>,
3977    ) -> Result<bool, SuperviseError> {
3978        let (reply_tx, reply_rx) = oneshot::channel();
3979        self.inner
3980            .commands
3981            .send(SupervisorCommand::SetEnabled {
3982                requester,
3983                enabled,
3984                reply: reply_tx,
3985            })
3986            .await
3987            .map_err(|_| SuperviseError::CommandClosed {
3988                module_id: self.inner.module_id.clone(),
3989            })?;
3990        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3991            module_id: self.inner.module_id.clone(),
3992        })?
3993    }
3994
3995    /// The current process's protocol, or the configured protocol when down.
3996    /// A rescan stores the next launch spec without changing how an existing
3997    /// process registers, serves routes, is probed, or exits.
3998    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3999        let configured = self
4000            .inner
4001            .configuration
4002            .lock()
4003            .map_err(|_| SuperviseError::StatePoisoned {
4004                module_id: Some(self.inner.module_id.clone()),
4005            })?
4006            .spec
4007            .protocol;
4008        let state = lock_snapshot(&self.inner.snapshot)?;
4009        Ok(state.spawned_protocol.unwrap_or(configured))
4010    }
4011
4012    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4013        let configuration =
4014            self.inner
4015                .configuration
4016                .lock()
4017                .map_err(|_| SuperviseError::StatePoisoned {
4018                    module_id: Some(self.inner.module_id.clone()),
4019                })?;
4020        Ok((configuration.spec.clone(), configuration.health.clone()))
4021    }
4022
4023    /// Replace this module's launch spec, keeping its health and drain policy,
4024    /// the way a rescan does for a changed config entry. The running process is
4025    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4026    #[cfg(any(test, feature = "test-support"))]
4027    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4028        let (_, health) = self.configuration()?;
4029        let drain_timeout_ms = u64::try_from(
4030            self.inner
4031                .effective_drain_timeout
4032                .lock()
4033                .unwrap_or_else(|poisoned| poisoned.into_inner())
4034                .as_millis(),
4035        )
4036        .ok();
4037        self.update_configuration(spec, health, drain_timeout_ms)
4038            .await
4039    }
4040
4041    pub(crate) async fn update_configuration(
4042        &self,
4043        spec: ModuleSpec,
4044        health: HealthConfig,
4045        drain_timeout_ms: Option<u64>,
4046    ) -> Result<(), SuperviseError> {
4047        if spec.module_id != self.inner.module_id {
4048            return Err(SuperviseError::InvalidSpec {
4049                reason: "a supervised module's module_id cannot be changed".to_string(),
4050            });
4051        }
4052        validate_spec(&spec)?;
4053        let (reply_tx, reply_rx) = oneshot::channel();
4054        self.inner
4055            .commands
4056            .send(SupervisorCommand::UpdateConfiguration {
4057                spec: spec.clone(),
4058                health: health.clone(),
4059                drain_timeout_ms,
4060                reply: reply_tx,
4061            })
4062            .await
4063            .map_err(|_| SuperviseError::CommandClosed {
4064                module_id: self.inner.module_id.clone(),
4065            })?;
4066        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4067            module_id: self.inner.module_id.clone(),
4068        })?;
4069        let mut configuration =
4070            self.inner
4071                .configuration
4072                .lock()
4073                .map_err(|_| SuperviseError::StatePoisoned {
4074                    module_id: Some(self.inner.module_id.clone()),
4075                })?;
4076        configuration.spec = spec;
4077        configuration.health = health;
4078        Ok(())
4079    }
4080}
4081
4082impl Drop for SupervisedModuleInner {
4083    fn drop(&mut self) {
4084        let Ok(mut monitor) = self.monitor.lock() else {
4085            return;
4086        };
4087        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4088            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4089                state.state = ModuleState::Stopped;
4090                clear_current_process_facts(state);
4091            });
4092            monitor.abort();
4093        }
4094        let _ = monitor.take();
4095    }
4096}
4097
4098#[derive(Debug)]
4099enum SupervisorCommand {
4100    Drain {
4101        requester: Option<Principal>,
4102        reply: oneshot::Sender<Result<(), SuperviseError>>,
4103    },
4104    Retire {
4105        requester: Option<Principal>,
4106        reply: oneshot::Sender<Result<(), SuperviseError>>,
4107    },
4108    Restart {
4109        requester: Option<Principal>,
4110        /// Operator override for this one restart's drain budget, in ms. `None`
4111        /// uses the module's configured/default budget; `Some(0)` cuts
4112        /// immediately (wedge bounce: a stuck request never settles, so
4113        /// waiting only delays recovery).
4114        drain_timeout_ms: Option<u64>,
4115        /// The module's `spawn_generation` when the request was received, before
4116        /// it waited in the command queue. A queued restart whose module has
4117        /// since spawned a newer process is already satisfied (see the handler).
4118        received_at_generation: u64,
4119        /// When the request entered the command queue, so the handler can log
4120        /// how long it waited behind the loop's other work.
4121        queued_at: Instant,
4122        reply: oneshot::Sender<Result<(), SuperviseError>>,
4123    },
4124    Reload {
4125        requester: Option<Principal>,
4126        reply: oneshot::Sender<Result<(), SuperviseError>>,
4127    },
4128    SetEnabled {
4129        requester: Option<Principal>,
4130        enabled: bool,
4131        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4132    },
4133    UpdateConfiguration {
4134        spec: ModuleSpec,
4135        health: HealthConfig,
4136        /// Per-module drain override from the new config; `None` re-resolves to
4137        /// the supervisor-wide default.
4138        drain_timeout_ms: Option<u64>,
4139        reply: oneshot::Sender<()>,
4140    },
4141    Swap {
4142        requester: Option<Principal>,
4143        /// How long the candidate may take to register and declare itself
4144        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4145        ready_timeout: Option<Duration>,
4146        /// Answered at cutover or failure; the incumbent's drain follows.
4147        reply: oneshot::Sender<Result<(), SuperviseError>>,
4148    },
4149}
4150
4151#[derive(Debug)]
4152pub enum SuperviseError {
4153    InvalidSpec {
4154        reason: String,
4155    },
4156    Spawn {
4157        program: PathBuf,
4158        source: io::Error,
4159        cgroup_path: Option<PathBuf>,
4160    },
4161    Cgroup {
4162        module_id: String,
4163        source: io::Error,
4164    },
4165    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4166    /// than spawn a reserved module without its identity binding.
4167    LaunchNonce {
4168        reason: String,
4169    },
4170    Wait {
4171        module_id: String,
4172        source: io::Error,
4173    },
4174    Kill {
4175        module_id: String,
4176        source: io::Error,
4177    },
4178    Forwarding(ForwardingError),
4179    Registry(RegistryError),
4180    ReloadUnavailable {
4181        module_id: String,
4182        reason: String,
4183    },
4184    /// An operator restart/reload was requested for a module that is currently
4185    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4186    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4187    /// by a restart, so these commands are rejected instead of re-enabling it.
4188    Disabled {
4189        module_id: String,
4190    },
4191    ReloadFailed {
4192        module_id: String,
4193        reason: String,
4194    },
4195    RegistrationStillActive {
4196        module_id: String,
4197        waited: Duration,
4198    },
4199    StatePoisoned {
4200        module_id: Option<String>,
4201    },
4202    CommandClosed {
4203        module_id: String,
4204    },
4205    /// A restart or reload arrived while a swap's candidate was warming. The
4206    /// swap owns the module until it cuts over or fails; a stop or disable
4207    /// would have aborted it instead.
4208    SwapInProgress {
4209        module_id: String,
4210    },
4211    /// A swap was refused before anything was spawned.
4212    SwapRefused {
4213        module_id: String,
4214        reason: SwapRefusal,
4215    },
4216    /// A swap spawned a candidate and gave up on it. The candidate has been
4217    /// killed and its slot freed; the incumbent was left serving and was never
4218    /// drained, except in the one `CutoverLost` case described on that arm.
4219    SwapFailed {
4220        module_id: String,
4221        arm: SwapFailureArm,
4222        detail: String,
4223        /// How the candidate exited, when it exited on its own before the
4224        /// supervisor gave up on it.
4225        candidate_exit: Option<ExitReport>,
4226    },
4227}
4228
4229/// Why a swap was refused before a candidate was spawned.
4230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4231pub enum SwapRefusal {
4232    /// The module's config does not declare `overlap: "safe"`.
4233    OverlapExclusive,
4234    /// The module is not registered, so there is no incumbent to keep serving
4235    /// and nothing a swap would improve on; a plain restart is the tool.
4236    NotRegistered,
4237    /// The module does not speak the subc wire, so a candidate could never
4238    /// register or declare itself ready.
4239    ProtocolNone,
4240    /// The supervisor lacks the forwarding table (to cut routes over) or the
4241    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4242    NotConfigured,
4243    /// A swap is already open for this module.
4244    AlreadySwapping,
4245}
4246
4247impl SwapRefusal {
4248    pub fn as_str(self) -> &'static str {
4249        match self {
4250            Self::OverlapExclusive => "overlap_exclusive",
4251            Self::NotRegistered => "not_registered",
4252            Self::ProtocolNone => "protocol_none",
4253            Self::NotConfigured => "not_configured",
4254            Self::AlreadySwapping => "already_swapping",
4255        }
4256    }
4257}
4258
4259/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4260/// serving and undrained; see `CutoverLost`.
4261#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4262pub enum SwapFailureArm {
4263    /// The candidate process could not be started.
4264    SpawnFailed,
4265    /// The candidate did not register within the readiness budget.
4266    NeverRegistered,
4267    /// The candidate registered but did not declare itself ready in time.
4268    NeverReady,
4269    /// The candidate exited before cutover.
4270    CandidateExited,
4271    /// The candidate declared itself ready but failed its health probe.
4272    CandidateUnhealthy,
4273    /// An operator stop, disable or retire arrived while the candidate warmed.
4274    /// The candidate was killed and the operator's command then carried out on
4275    /// the incumbent.
4276    Interrupted,
4277    /// The candidate's connection closed at the moment of cutover. If it
4278    /// closed before forwarding moved, the incumbent is untouched. If it closed
4279    /// between the forwarding and registry halves of cutover, forwarding can no
4280    /// longer route to the incumbent, so the module is restarted plainly.
4281    CutoverLost,
4282}
4283
4284impl SwapFailureArm {
4285    pub fn as_str(self) -> &'static str {
4286        match self {
4287            Self::SpawnFailed => "spawn_failed",
4288            Self::NeverRegistered => "never_registered",
4289            Self::NeverReady => "never_ready",
4290            Self::CandidateExited => "candidate_exited",
4291            Self::CandidateUnhealthy => "candidate_unhealthy",
4292            Self::Interrupted => "interrupted",
4293            Self::CutoverLost => "cutover_lost",
4294        }
4295    }
4296}
4297
4298impl fmt::Display for SuperviseError {
4299    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4300        match self {
4301            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4302            Self::Spawn {
4303                program,
4304                source,
4305                cgroup_path: Some(cgroup_path),
4306            } => write!(
4307                f,
4308                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4309                cgroup_path.display(),
4310                program.display()
4311            ),
4312            Self::Spawn {
4313                program,
4314                source,
4315                cgroup_path: None,
4316            } => write!(
4317                f,
4318                "failed to spawn module '{}': {source}",
4319                program.display()
4320            ),
4321            Self::Cgroup { module_id, source } => {
4322                write!(
4323                    f,
4324                    "failed to prepare cgroup for module '{module_id}': {source}"
4325                )
4326            }
4327            Self::LaunchNonce { reason } => {
4328                write!(
4329                    f,
4330                    "failed to generate reserved-module launch nonce: {reason}"
4331                )
4332            }
4333            Self::Wait { module_id, source } => {
4334                write!(f, "failed to wait for module '{module_id}': {source}")
4335            }
4336            Self::Kill { module_id, source } => {
4337                write!(f, "failed to kill module '{module_id}': {source}")
4338            }
4339            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4340            Self::Registry(err) => write!(f, "registry error: {err}"),
4341            Self::ReloadUnavailable { module_id, reason } => {
4342                write!(f, "reload unavailable for module '{module_id}': {reason}")
4343            }
4344            Self::Disabled { module_id } => {
4345                write!(
4346                    f,
4347                    "module '{module_id}' is disabled; enable it before restart or reload"
4348                )
4349            }
4350            Self::ReloadFailed { module_id, reason } => {
4351                write!(f, "reload failed for module '{module_id}': {reason}")
4352            }
4353            Self::RegistrationStillActive { module_id, waited } => write!(
4354                f,
4355                "module '{module_id}' registration remained active after waiting {waited:?}"
4356            ),
4357            Self::StatePoisoned { module_id } => match module_id {
4358                Some(module_id) => {
4359                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4360                }
4361                None => write!(f, "supervisor state was poisoned"),
4362            },
4363            Self::CommandClosed { module_id } => {
4364                write!(
4365                    f,
4366                    "supervisor command channel for module '{module_id}' is closed"
4367                )
4368            }
4369            Self::SwapInProgress { module_id } => write!(
4370                f,
4371                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4372            ),
4373            Self::SwapRefused { module_id, reason } => match reason {
4374                SwapRefusal::OverlapExclusive => write!(
4375                    f,
4376                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4377                ),
4378                SwapRefusal::NotRegistered => write!(
4379                    f,
4380                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4381                ),
4382                SwapRefusal::ProtocolNone => write!(
4383                    f,
4384                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4385                ),
4386                SwapRefusal::NotConfigured => write!(
4387                    f,
4388                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4389                ),
4390                SwapRefusal::AlreadySwapping => {
4391                    write!(f, "module '{module_id}' is already being swapped")
4392                }
4393            },
4394            Self::SwapFailed {
4395                module_id,
4396                arm,
4397                detail,
4398                ..
4399            } => write!(
4400                f,
4401                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4402                arm.as_str()
4403            ),
4404        }
4405    }
4406}
4407
4408impl Error for SuperviseError {
4409    fn source(&self) -> Option<&(dyn Error + 'static)> {
4410        match self {
4411            Self::Spawn { source, .. }
4412            | Self::Cgroup { source, .. }
4413            | Self::Wait { source, .. }
4414            | Self::Kill { source, .. } => Some(source),
4415            Self::Forwarding(err) => Some(err),
4416            Self::Registry(err) => Some(err),
4417            Self::LaunchNonce { .. }
4418            | Self::InvalidSpec { .. }
4419            | Self::ReloadUnavailable { .. }
4420            | Self::Disabled { .. }
4421            | Self::ReloadFailed { .. }
4422            | Self::RegistrationStillActive { .. }
4423            | Self::StatePoisoned { .. }
4424            | Self::CommandClosed { .. }
4425            | Self::SwapInProgress { .. }
4426            | Self::SwapRefused { .. }
4427            | Self::SwapFailed { .. } => None,
4428        }
4429    }
4430}
4431
4432pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4433    if spec.module_id.trim().is_empty() {
4434        return Err(SuperviseError::InvalidSpec {
4435            reason: "module_id must not be empty".to_string(),
4436        });
4437    }
4438
4439    Ok(())
4440}
4441
4442#[derive(Debug, Default)]
4443struct HealthProbeRuntime {
4444    configured_health: Option<HealthConfig>,
4445    registered_connection: Option<crate::ConnectionId>,
4446    advertised: bool,
4447    next_probe_at: Option<Instant>,
4448    probe_index: u64,
4449}
4450
4451fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4452    lock_snapshot(snapshot)
4453        .ok()
4454        .and_then(|state| state.spawned_protocol)
4455        .unwrap_or(spec.protocol)
4456}
4457
4458impl HealthProbeRuntime {
4459    fn refresh_registration(
4460        &mut self,
4461        spec: &ModuleSpec,
4462        runtime: &SupervisorRuntimeConfig,
4463        registry: &Registry,
4464        snapshot: &SharedSnapshot,
4465    ) {
4466        if self.configured_health.as_ref() != Some(&runtime.health) {
4467            self.configured_health = Some(runtime.health.clone());
4468            self.next_probe_at = None;
4469            self.registered_connection = None;
4470            self.probe_index = 0;
4471        }
4472        // A non-wire process never registers. Only an explicitly configured
4473        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4474        // health failure for that kind of process.
4475        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4476            self.registered_connection = None;
4477            self.advertised = runtime.health.http.is_some();
4478            if !self.advertised {
4479                self.next_probe_at = None;
4480                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4481                    let unknown = ModuleHealthStatus::default();
4482                    if state.health != unknown {
4483                        state.health = unknown;
4484                    }
4485                });
4486            } else if self.next_probe_at.is_none() {
4487                self.next_probe_at = Some(
4488                    Instant::now()
4489                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4490                );
4491            }
4492            return;
4493        }
4494
4495        let registration = match registry.get_module(&spec.module_id) {
4496            Ok(registration) => registration,
4497            Err(err) => {
4498                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4499                self.advertised = false;
4500                self.next_probe_at = None;
4501                return;
4502            }
4503        };
4504
4505        let Some(registration) = registration else {
4506            self.registered_connection = None;
4507            self.advertised = false;
4508            self.next_probe_at = None;
4509            return;
4510        };
4511
4512        let advertised = registration
4513            .control_ops
4514            .iter()
4515            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4516        if !advertised {
4517            self.registered_connection = Some(registration.connection_id);
4518            self.advertised = false;
4519            self.next_probe_at = None;
4520            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4521                if state.health.status != SupervisorHealthStatus::Unknown
4522                    || state.health.consecutive_failures != 0
4523                    || state.health.last_probe_ms.is_some()
4524                    || state.health.detail.is_some()
4525                    || state.health.metrics.is_some()
4526                {
4527                    state.health.status = SupervisorHealthStatus::Unknown;
4528                    state.health.consecutive_failures = 0;
4529                    state.health.last_probe_ms = None;
4530                    state.health.detail = None;
4531                    state.health.metrics = None;
4532                }
4533            });
4534            return;
4535        }
4536
4537        let reregistered = self.registered_connection != Some(registration.connection_id);
4538        self.registered_connection = Some(registration.connection_id);
4539        self.advertised = true;
4540        if reregistered || self.next_probe_at.is_none() {
4541            self.probe_index = 0;
4542            self.next_probe_at = Some(
4543                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4544            );
4545            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4546                state.health.status = SupervisorHealthStatus::Unknown;
4547                state.health.consecutive_failures = 0;
4548                state.health.detail = None;
4549                state.health.metrics = None;
4550            });
4551        }
4552    }
4553
4554    fn wake_after(&self) -> Option<Duration> {
4555        if !self.advertised {
4556            return None;
4557        }
4558        self.next_probe_at
4559            .map(|next| next.saturating_duration_since(Instant::now()))
4560    }
4561
4562    fn due(&self) -> bool {
4563        self.advertised
4564            && self
4565                .next_probe_at
4566                .is_some_and(|next| Instant::now() >= next)
4567    }
4568
4569    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4570        self.probe_index = self.probe_index.wrapping_add(1);
4571        self.next_probe_at = Some(
4572            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4573        );
4574    }
4575}
4576
4577/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4578///
4579/// This was a struct with a single `message: String`, and every one of the
4580/// fifteen construction sites collapsed into it. Each site knows exactly what it
4581/// saw -- the lane is gone, the module did not answer in time, the module
4582/// answered with the wrong thing -- and `handle_health_probe_failure` then
4583/// treated all of them identically: increment a counter, compare to a threshold,
4584/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4585/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4586///
4587/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4588///
4589/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4590///   answer on it again.
4591/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4592///   AND with a perfectly healthy one that lost a CPU race -- which is what
4593///   happens under machine load, and is how this supervisor killed a healthy
4594///   module three times in one day.
4595/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4596///   Restarting on it is defensible, but it is not the silence case and should
4597///   never be counted as one.
4598/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4599///   anything, so it cannot be evidence about the module at all.
4600///
4601/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4602/// one that fires most often, and while every variant collapsed into one string
4603/// it carried the same weight as the strongest.
4604///
4605/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4606/// DESIGN and a reader stopping at it gets the build backwards: the restart
4607/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4608/// probes still increment the failure streak and drive escalation at the
4609/// threshold (see `is_proof_of_death` below for why that is deliberate and
4610/// what gates the change). Absence of evidence restarts modules today.
4611#[derive(Debug)]
4612enum HealthProbeEvidence {
4613    /// The module's control lane is gone. Proof of death.
4614    LaneDead,
4615    /// No reply within the deadline. Proves nothing about the module's state.
4616    NoAnswer,
4617    /// The module replied, but not with a usable health report. Proves it is alive.
4618    BadAnswer,
4619    /// The daemon could not ask. Says nothing about the module.
4620    Misconfigured,
4621}
4622
4623#[derive(Debug)]
4624struct HealthProbeError {
4625    evidence: HealthProbeEvidence,
4626    message: String,
4627}
4628
4629impl HealthProbeError {
4630    fn lane_dead(message: impl Into<String>) -> Self {
4631        Self::with(HealthProbeEvidence::LaneDead, message)
4632    }
4633
4634    fn no_answer(message: impl Into<String>) -> Self {
4635        Self::with(HealthProbeEvidence::NoAnswer, message)
4636    }
4637
4638    fn bad_answer(message: impl Into<String>) -> Self {
4639        Self::with(HealthProbeEvidence::BadAnswer, message)
4640    }
4641
4642    fn misconfigured(message: impl Into<String>) -> Self {
4643        Self::with(HealthProbeEvidence::Misconfigured, message)
4644    }
4645
4646    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4647        Self {
4648            evidence,
4649            message: message.into(),
4650        }
4651    }
4652
4653    /// Whether this observation is proof the module cannot serve.
4654    ///
4655    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4656    /// variant that fires under CPU starvation, and treating it as proof is the
4657    /// defect this enum exists to make impossible to reintroduce silently.
4658    ///
4659    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4660    /// to restart also needs a bound for the case it excludes -- a genuinely
4661    /// wedged module, alive but never answering -- and that bound must come from
4662    /// the distribution of real late-answer latencies, which nothing measures
4663    /// yet. Landing the classification first makes the later change a one-line
4664    /// decision against evidence that already exists, rather than two unproven
4665    /// changes at once.
4666    #[allow(dead_code)]
4667    fn is_proof_of_death(&self) -> bool {
4668        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4669    }
4670
4671    /// Short stable label for logs and the health snapshot.
4672    ///
4673    /// An operator reading `ck health` currently cannot tell "the module is gone"
4674    /// from "the module did not answer in five seconds", because both render as
4675    /// prose in the same field. These labels are what make the two
4676    /// distinguishable at a glance, and they are what a later restart-policy
4677    /// change will be argued from.
4678    fn label(&self) -> &'static str {
4679        match self.evidence {
4680            HealthProbeEvidence::LaneDead => "lane-dead",
4681            HealthProbeEvidence::NoAnswer => "no-answer",
4682            HealthProbeEvidence::BadAnswer => "bad-answer",
4683            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4684        }
4685    }
4686}
4687
4688impl fmt::Display for HealthProbeError {
4689    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4690        f.write_str(&self.message)
4691    }
4692}
4693
4694async fn run_health_probe_cycle(
4695    spec: &ModuleSpec,
4696    runtime: &SupervisorRuntimeConfig,
4697    registry: &Registry,
4698    process_liveness: &SupervisorProcessLiveness,
4699    snapshot: &SharedSnapshot,
4700    child: &mut Option<SupervisedChild>,
4701) {
4702    let now_ms = unix_ms_now();
4703    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4704        .then_some(runtime.health.http.as_deref())
4705        .flatten();
4706    let result = match http {
4707        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4708        None => probe_module_health(&spec.module_id, runtime, None).await,
4709    };
4710    match result {
4711        Ok(report) => {
4712            handle_health_report(
4713                spec,
4714                runtime,
4715                registry,
4716                process_liveness,
4717                snapshot,
4718                child,
4719                report,
4720                now_ms,
4721            )
4722            .await;
4723        }
4724        Err(err) => {
4725            if http.is_some() {
4726                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4727                    state.health.status = SupervisorHealthStatus::Failing;
4728                });
4729            }
4730            handle_health_probe_failure(
4731                spec,
4732                runtime,
4733                registry,
4734                process_liveness,
4735                snapshot,
4736                child,
4737                err,
4738                now_ms,
4739            )
4740            .await;
4741        }
4742    }
4743}
4744
4745pub(crate) struct HttpProbeTarget<'a> {
4746    address: std::net::SocketAddr,
4747    localhost: bool,
4748    authority: &'a str,
4749    path: String,
4750}
4751
4752/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4753/// or TLS. A URL cannot turn a local health check into an outbound connection.
4754pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4755    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4756        return Err("must not contain whitespace, controls, or a fragment".into());
4757    }
4758    let rest = url
4759        .strip_prefix("http://")
4760        .ok_or("must use plain http://")?;
4761    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4762    let (authority, suffix) = rest.split_at(split);
4763    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4764        ("::1", rest)
4765    } else {
4766        let split = authority.find(':').unwrap_or(authority.len());
4767        authority.split_at(split)
4768    };
4769    let ip: std::net::IpAddr = match host {
4770        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4771        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4772        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4773    };
4774    let port = if port.is_empty() {
4775        80
4776    } else {
4777        port.strip_prefix(':')
4778            .and_then(|p| p.parse::<u16>().ok())
4779            .filter(|p| *p > 0)
4780            .ok_or("must have a valid nonzero TCP port")?
4781    };
4782    let path = if suffix.is_empty() {
4783        "/".into()
4784    } else if suffix.starts_with('?') {
4785        format!("/{suffix}")
4786    } else {
4787        suffix.into()
4788    };
4789    Ok(HttpProbeTarget {
4790        address: std::net::SocketAddr::new(ip, port),
4791        localhost: host == "localhost",
4792        authority,
4793        path,
4794    })
4795}
4796
4797async fn probe_http_health(
4798    url: &str,
4799    deadline: Duration,
4800) -> Result<HealthReport, HealthProbeError> {
4801    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4802    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4803    // Keep partial diagnostics outside the timed future so cancellation does
4804    // not discard a status line or body bytes already received.
4805    let mut response_status = String::new();
4806    let mut body = Vec::new();
4807    let probe = async {
4808        // Resolve localhost ourselves so a hosts-file override cannot turn
4809        // this into an outbound request, while IPv6-only local servers work.
4810        let connection = match tokio::net::TcpStream::connect(target.address).await {
4811            Err(_) if target.localhost => {
4812                tokio::net::TcpStream::connect((
4813                    std::net::Ipv6Addr::LOCALHOST,
4814                    target.address.port(),
4815                ))
4816                .await
4817            }
4818            result => result,
4819        };
4820        let mut stream = connection.map_err(|error| {
4821            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4822        })?;
4823        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4824            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4825        let mut reader = BufReader::new(stream);
4826        let mut budget = 16 * 1024;
4827        let status = http_line(&mut reader, &mut budget).await?;
4828        let mut words = status.split_ascii_whitespace();
4829        let version = words.next();
4830        let code = words
4831            .next()
4832            .filter(|word| word.len() == 3)
4833            .and_then(|word| word.parse::<u16>().ok());
4834        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4835            || !code.is_some_and(|code| (100..600).contains(&code))
4836        {
4837            return Err(HealthProbeError::bad_answer(format!(
4838                "invalid HTTP status: {status}"
4839            )));
4840        }
4841        let code = code.expect("validated status code");
4842        response_status = status.clone();
4843        let mut length = None;
4844        let mut chunked = false;
4845        loop {
4846            let line = http_line(&mut reader, &mut budget).await?;
4847            if line.is_empty() {
4848                break;
4849            }
4850            if let Some((name, value)) = line.split_once(':') {
4851                if name.eq_ignore_ascii_case("content-length") {
4852                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4853                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4854                    })?);
4855                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4856                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4857                }
4858            }
4859        }
4860        if chunked {
4861            while body.len() < 200 {
4862                let line = http_line(&mut reader, &mut budget).await?;
4863                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4864                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4865                if size == 0 {
4866                    break;
4867                }
4868                let count = size.min((200 - body.len()) as u64) as usize;
4869                let start = body.len();
4870                (&mut reader)
4871                    .take(count as u64)
4872                    .read_to_end(&mut body)
4873                    .await
4874                    .map_err(|error| {
4875                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4876                    })?;
4877                if body.len() - start != count {
4878                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4879                }
4880                if size > count as u64 || body.len() == 200 {
4881                    break;
4882                }
4883                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4884                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4885                }
4886            }
4887        } else {
4888            reader
4889                .take(length.unwrap_or(200).min(200))
4890                .read_to_end(&mut body)
4891                .await
4892                .map_err(|error| {
4893                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4894                })?;
4895        }
4896        if (200..300).contains(&code) {
4897            Ok(HealthReport::ok())
4898        } else {
4899            Err(HealthProbeError::bad_answer(
4900                "HTTP health endpoint returned non-2xx",
4901            ))
4902        }
4903    };
4904    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4905        Err(HealthProbeError::no_answer(format!(
4906            "HTTP probe timed out after {deadline:?}"
4907        )))
4908    });
4909    if let Err(error) = &mut result {
4910        if !response_status.is_empty() {
4911            error.message = format!(
4912                "{}; {response_status}: {}",
4913                error.message,
4914                String::from_utf8_lossy(&body)
4915            );
4916        }
4917    }
4918    result
4919}
4920
4921async fn http_line(
4922    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4923    remaining: &mut usize,
4924) -> Result<String, HealthProbeError> {
4925    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4926    let mut line = Vec::new();
4927    (&mut *reader)
4928        .take(*remaining as u64)
4929        .read_until(b'\n', &mut line)
4930        .await
4931        .map_err(|error| {
4932            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4933        })?;
4934    *remaining -= line.len();
4935    if !line.ends_with(b"\r\n") {
4936        return Err(HealthProbeError::bad_answer(
4937            "HTTP headers are incomplete or exceed 16 KiB",
4938        ));
4939    }
4940    line.truncate(line.len() - 2);
4941    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4942}
4943
4944async fn probe_module_health(
4945    module_id: &str,
4946    runtime: &SupervisorRuntimeConfig,
4947    drain_deadline: Option<Instant>,
4948) -> Result<HealthReport, HealthProbeError> {
4949    let Some(forwarding) = runtime.forwarding.as_ref() else {
4950        return Err(HealthProbeError::misconfigured(
4951            "supervisor was not configured with a forwarding table",
4952        ));
4953    };
4954    let probe_started_at = Instant::now();
4955    let mut deadline = probe_started_at + runtime.health.deadline;
4956    if let Some(drain_deadline) = drain_deadline {
4957        deadline = deadline.min(drain_deadline);
4958    }
4959    let pending = if drain_deadline.is_some() {
4960        forwarding.begin_drain_health_probe_rpc_for(
4961            module_id,
4962            MODULE_CONTROL_OP_HEALTH_CHECK,
4963            probe_started_at,
4964            deadline,
4965        )
4966    } else {
4967        forwarding.begin_health_probe_rpc_for(
4968            module_id,
4969            MODULE_CONTROL_OP_HEALTH_CHECK,
4970            probe_started_at,
4971            deadline,
4972        )
4973    }
4974    .map_err(|err| {
4975        // The endpoint is not registered, so there is no live control lane to
4976        // ask. That is the module being absent, not slow.
4977        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4978    })?;
4979    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4980}
4981
4982/// [`probe_module_health`] for one endpoint rather than the id's active one.
4983///
4984/// A swap probes two processes that no by-id lookup reaches: its candidate
4985/// before cutover, and its superseded incumbent (for busy gauges) while the
4986/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4987/// bounds the by-id drain probe.
4988async fn probe_endpoint_health(
4989    endpoint: crate::ModuleEndpointId,
4990    runtime: &SupervisorRuntimeConfig,
4991    deadline_cap: Option<Instant>,
4992) -> Result<HealthReport, HealthProbeError> {
4993    let Some(forwarding) = runtime.forwarding.as_ref() else {
4994        return Err(HealthProbeError::misconfigured(
4995            "supervisor was not configured with a forwarding table",
4996        ));
4997    };
4998    let probe_started_at = Instant::now();
4999    let mut deadline = probe_started_at + runtime.health.deadline;
5000    if let Some(cap) = deadline_cap {
5001        deadline = deadline.min(cap);
5002    }
5003    let pending = forwarding
5004        .begin_endpoint_health_probe_rpc_for(
5005            endpoint,
5006            MODULE_CONTROL_OP_HEALTH_CHECK,
5007            probe_started_at,
5008            deadline,
5009        )
5010        .map_err(|err| {
5011            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5012        })?;
5013    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5014}
5015
5016/// Send a begun health probe and classify its answer.
5017async fn await_health_probe(
5018    forwarding: &ForwardingTable,
5019    pending: PendingModuleControlRpc,
5020    deadline: Instant,
5021    probe_budget: Duration,
5022) -> Result<HealthReport, HealthProbeError> {
5023    let PendingModuleControlRpc {
5024        endpoint,
5025        module_sink,
5026        negotiated_ver,
5027        corr,
5028        receiver,
5029    } = pending;
5030    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5031        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5032    })?;
5033    let frame = Frame::build_with_version(
5034        negotiated_ver,
5035        FrameType::Request,
5036        control_flags(),
5037        0,
5038        0,
5039        corr,
5040        body,
5041    )
5042    .map_err(|err| {
5043        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5044    })?;
5045
5046    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5047    // blocks waiting for capacity when the module's egress queue is full, and an
5048    // unbounded await here freezes the whole supervision actor (it stops polling
5049    // Child::wait and supervisor commands), making the module unrecoverable
5050    // in-band. On timeout the probe fails like any transport failure.
5051    match timeout_at(deadline, module_sink.send(frame)).await {
5052        Ok(Ok(())) => {}
5053        Ok(Err(err)) => {
5054            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5055            // A closed sink means the module's egress channel is gone -- the
5056            // receiving half is dropped when its connection tears down. Proof.
5057            return Err(HealthProbeError::lane_dead(format!(
5058                "failed to send health.check: {err}"
5059            )));
5060        }
5061        Err(_elapsed) => {
5062            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5063            // A full egress queue means the module is not draining its socket, which
5064            // is consistent with a wedged module AND with one whose reader is merely
5065            // starved. Silence, not proof.
5066            return Err(HealthProbeError::no_answer(
5067                "health.check send timed out before enqueue (module egress full)",
5068            ));
5069        }
5070    }
5071
5072    match timeout_at(deadline, receiver).await {
5073        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5074        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5075        // and those prove it is alive even though the probe failed.
5076        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5077            response.health_report().ok_or_else(|| {
5078                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5079            })
5080        }
5081        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5082            format!("health.check rejected: {}", body.message),
5083        )),
5084        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5085            Err(HealthProbeError::lane_dead(message))
5086        }
5087        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5088            Err(HealthProbeError::bad_answer(message))
5089        }
5090        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5091            Err(HealthProbeError::bad_answer(format!(
5092                "expected module-control op '{expected}', got '{actual}'"
5093            )))
5094        }
5095        // A reply that crosses the deadline before this waiter observes it is
5096        // still proof of life. The forwarding path records its end-to-end latency
5097        // before delivering this classification.
5098        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5099            "module answered health.check after its daemon deadline",
5100        )),
5101        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5102            "health.check waiter was canceled before the module responded",
5103        )),
5104        Err(_) => {
5105            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5106            Err(HealthProbeError::no_answer(format!(
5107                "module did not answer health.check within {probe_budget:?}"
5108            )))
5109        }
5110    }
5111}
5112
5113#[allow(clippy::too_many_arguments)]
5114async fn handle_health_report(
5115    spec: &ModuleSpec,
5116    runtime: &SupervisorRuntimeConfig,
5117    registry: &Registry,
5118    process_liveness: &SupervisorProcessLiveness,
5119    snapshot: &SharedSnapshot,
5120    child: &mut Option<SupervisedChild>,
5121    report: HealthReport,
5122    now_ms: u64,
5123) {
5124    let status = supervisor_health_status(report.status);
5125    let detail = report.detail.clone();
5126    let metrics = truncate_health_metrics(report.metrics);
5127    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5128        state.health.status = status;
5129        state.health.last_probe_ms = Some(now_ms);
5130        state.health.detail = detail.clone();
5131        state.health.metrics = metrics.clone();
5132        state.health.consecutive_failures = 0;
5133    });
5134
5135    let action = match report.status {
5136        HealthStatus::Ok => return,
5137        HealthStatus::Degraded => runtime.health.on_degraded,
5138        HealthStatus::Failing => runtime.health.on_failing,
5139    };
5140    apply_l3_health_action(
5141        spec,
5142        runtime,
5143        registry,
5144        process_liveness,
5145        snapshot,
5146        child,
5147        status,
5148        detail.as_deref(),
5149        action,
5150        now_ms,
5151    )
5152    .await;
5153}
5154
5155#[allow(clippy::too_many_arguments)]
5156async fn handle_health_probe_failure(
5157    spec: &ModuleSpec,
5158    runtime: &SupervisorRuntimeConfig,
5159    registry: &Registry,
5160    process_liveness: &SupervisorProcessLiveness,
5161    snapshot: &SharedSnapshot,
5162    child: &mut Option<SupervisedChild>,
5163    err: HealthProbeError,
5164    now_ms: u64,
5165) {
5166    let threshold = runtime.health.failure_threshold.max(1);
5167    let mut failures = 0;
5168    // Carry the evidence class into the operator-visible detail. Without it,
5169    // "module did not answer within 5s" and "the control lane is gone" are two
5170    // prose strings in the same field, and the reader has to know the codebase to
5171    // tell which one is proof of anything.
5172    let detail = format!("[{}] {err}", err.label());
5173    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5174        // A failed wire probe invalidates the last report, even before the
5175        // restart threshold. HTTP probes already mark failures as Failing.
5176        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5177            state.health.status = SupervisorHealthStatus::Unknown;
5178        }
5179        state.health.last_probe_ms = Some(now_ms);
5180        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5181        state.health.detail = Some(detail.clone());
5182        state.health.metrics = None;
5183        failures = state.health.consecutive_failures;
5184    });
5185
5186    if failures < threshold {
5187        warn!(
5188            module_id = %spec.module_id,
5189            consecutive_failures = failures,
5190            threshold,
5191            evidence = err.label(),
5192            detail = %detail,
5193            "health.check probe failed"
5194        );
5195        return;
5196    }
5197
5198    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5199        state.state = ModuleState::Unresponsive;
5200        state.health.status = SupervisorHealthStatus::Unresponsive;
5201    });
5202    // The evidence class is logged at the kill site because this is the line an
5203    // operator reads after an unexplained restart. A streak of `no-answer` under
5204    // machine load is the known false-positive shape; a `lane-dead` is not.
5205    if runtime.health.critical {
5206        error!(
5207            module_id = %spec.module_id,
5208            status = "unresponsive",
5209            evidence = err.label(),
5210            detail = %detail,
5211            "critical module health alert"
5212        );
5213    } else {
5214        warn!(
5215            module_id = %spec.module_id,
5216            status = "unresponsive",
5217            evidence = err.label(),
5218            detail = %detail,
5219            "module health threshold breached"
5220        );
5221    }
5222    if let Err(err) = health_restart_child(
5223        spec,
5224        runtime,
5225        registry,
5226        process_liveness,
5227        snapshot,
5228        child,
5229        SupervisorHealthStatus::Unresponsive,
5230        Some(&detail),
5231        now_ms,
5232    )
5233    .await
5234    {
5235        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5236    }
5237}
5238
5239#[allow(clippy::too_many_arguments)]
5240async fn apply_l3_health_action(
5241    spec: &ModuleSpec,
5242    runtime: &SupervisorRuntimeConfig,
5243    registry: &Registry,
5244    process_liveness: &SupervisorProcessLiveness,
5245    snapshot: &SharedSnapshot,
5246    child: &mut Option<SupervisedChild>,
5247    status: SupervisorHealthStatus,
5248    detail: Option<&str>,
5249    action: HealthAction,
5250    now_ms: u64,
5251) {
5252    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5253    match action {
5254        HealthAction::Report => {
5255            info!(
5256                module_id = %spec.module_id,
5257                status = ?status,
5258                detail,
5259                "module reported non-ok health"
5260            );
5261        }
5262        HealthAction::Alert => {
5263            error!(
5264                module_id = %spec.module_id,
5265                status = ?status,
5266                detail,
5267                "module health alert"
5268            );
5269        }
5270        HealthAction::Restart => {
5271            if let Err(err) = health_restart_child(
5272                spec,
5273                runtime,
5274                registry,
5275                process_liveness,
5276                snapshot,
5277                child,
5278                status,
5279                detail,
5280                now_ms,
5281            )
5282            .await
5283            {
5284                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5285            }
5286        }
5287    }
5288}
5289
5290#[allow(clippy::too_many_arguments)]
5291async fn health_restart_child(
5292    spec: &ModuleSpec,
5293    runtime: &SupervisorRuntimeConfig,
5294    registry: &Registry,
5295    process_liveness: &SupervisorProcessLiveness,
5296    snapshot: &SharedSnapshot,
5297    child: &mut Option<SupervisedChild>,
5298    status: SupervisorHealthStatus,
5299    detail: Option<&str>,
5300    now_ms: u64,
5301) -> Result<(), SuperviseError> {
5302    let (enabled, schedule) = {
5303        let mut state = lock_snapshot(snapshot)?;
5304        let enabled = state.enabled;
5305        let schedule = if enabled {
5306            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5307        } else {
5308            None
5309        };
5310        (enabled, schedule)
5311    };
5312
5313    if !enabled {
5314        return Err(SuperviseError::Disabled {
5315            module_id: spec.module_id.clone(),
5316        });
5317    }
5318
5319    if schedule.is_none() {
5320        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5321        error!(
5322            module_id = %spec.module_id,
5323            status = ?status,
5324            detail,
5325            max_restarts = runtime.restart_policy.max_restarts,
5326            window_secs = runtime.restart_policy.window.as_secs(),
5327            reason = %runtime.restart_policy.budget_exhausted_detail(),
5328            "health restart budget exhausted; marking module failed"
5329        );
5330        let stop_notice = begin_forwarding_drain_if_configured(
5331            spec,
5332            runtime,
5333            registry,
5334            snapshot,
5335            Some(true),
5336            RouteCloseReason::Disable,
5337        )
5338        .await?;
5339        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5340            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5341        })?;
5342        drain_optional_child(
5343            &spec.module_id,
5344            spec.protocol,
5345            stop_notice,
5346            registry,
5347            runtime.forwarding.as_deref(),
5348            snapshot,
5349            &runtime.terminal_ring,
5350            &runtime.spawn_events,
5351            child,
5352            runtime.drain_timeout,
5353            ModuleState::Failed,
5354            Some(true),
5355        )
5356        .await?;
5357        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5358        return Ok(());
5359    }
5360
5361    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5362    let mut restart_count = 0;
5363    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5364        restart_count = state.crash_restarts.len();
5365        state.state = ModuleState::Unresponsive;
5366        state.health.status = status;
5367        state.health.last_action = Some(HealthAction::Restart.to_string());
5368        state.health.last_action_ms = Some(now_ms);
5369    })?;
5370    warn!(
5371        module_id = %spec.module_id,
5372        status = ?status,
5373        detail,
5374        restart_count,
5375        restart_in_window = schedule.restart_in_window,
5376        delay_ms = schedule.delay.as_millis() as u64,
5377        "health-triggered module restart"
5378    );
5379
5380    let stop_notice = begin_forwarding_drain_if_configured(
5381        spec,
5382        runtime,
5383        registry,
5384        snapshot,
5385        Some(true),
5386        RouteCloseReason::Restart,
5387    )
5388    .await?;
5389    drain_optional_child(
5390        &spec.module_id,
5391        spec.protocol,
5392        stop_notice,
5393        registry,
5394        runtime.forwarding.as_deref(),
5395        snapshot,
5396        &runtime.terminal_ring,
5397        &runtime.spawn_events,
5398        child,
5399        runtime.drain_timeout,
5400        ModuleState::Restarting,
5401        Some(true),
5402    )
5403    .await?;
5404    schedule_respawn(
5405        runtime,
5406        snapshot,
5407        &spec.module_id,
5408        schedule.delay,
5409        RespawnKind::Spawn,
5410    )
5411}
5412
5413fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5414    if let Some(reply) = runtime
5415        .deferred_reload_reply
5416        .lock()
5417        .unwrap_or_else(|p| p.into_inner())
5418        .take()
5419    {
5420        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5421            module_id: module_id.to_string(),
5422            reason: reason.to_string(),
5423        }));
5424    }
5425}
5426
5427fn schedule_respawn(
5428    runtime: &SupervisorRuntimeConfig,
5429    snapshot: &SharedSnapshot,
5430    module_id: &str,
5431    delay: Duration,
5432    kind: RespawnKind,
5433) -> Result<(), SuperviseError> {
5434    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5435    update_snapshot(snapshot, Some(module_id), |state| {
5436        state.respawn_pending = true
5437    })?;
5438    *runtime
5439        .scheduled_respawn
5440        .lock()
5441        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5442        deadline: Instant::now() + delay,
5443        kind,
5444    });
5445    Ok(())
5446}
5447
5448fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5449    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5450        state.health.last_action = Some(action);
5451        state.health.last_action_ms = Some(now_ms);
5452    });
5453}
5454
5455fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5456    match status {
5457        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5458        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5459        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5460    }
5461}
5462
5463/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5464/// returned to every `supervisor.list` and `supervisor.health` caller.
5465///
5466/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5467/// path: that request exists to return a module's complete metrics object, and
5468/// `ck health <module-id>` documents it as the way to see what the cached view
5469/// truncates. The asymmetry is the feature.
5470///
5471/// So a new caller must decide which side it is on rather than assume the cap is
5472/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5473/// the truncation that path exists to avoid.
5474fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5475    let metrics = metrics?;
5476    match serde_json::to_vec(&metrics) {
5477        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5478            "truncated": true,
5479            "original_bytes": encoded.len(),
5480        })),
5481        Ok(_) | Err(_) => Some(metrics),
5482    }
5483}
5484
5485/// Spread health probes so a fleet-wide restart does not converge them.
5486///
5487/// The delay is derived from the module id and probe index rather than a random
5488/// source, so it is deterministic per module: a module keeps its own offset
5489/// across daemon restarts instead of re-rolling into a collision.
5490fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5491    if cadence.is_zero() {
5492        return Duration::ZERO;
5493    }
5494    let cadence_ms = cadence.as_millis() as u64;
5495    // This early return is REDUNDANT, deliberately, and a mutation run will show
5496    // it surviving removal. Recording why here so the next person to notice does
5497    // not have to re-derive it:
5498    //
5499    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5500    //   a zero cadence and builds the Duration from whole milliseconds, so a
5501    //   sub-millisecond cadence cannot come from config.
5502    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5503    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5504    //   -- exactly what this returns.
5505    //
5506    // Kept as a guard against a future widening of the config parser (accepting
5507    // microseconds, say), which would make the sub-millisecond case reachable.
5508    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5509    // divides by zero. Remove this and nothing changes.
5510    if cadence_ms == 0 {
5511        return cadence;
5512    }
5513    // Note that this never returns less than one cadence, including for the FIRST
5514    // probe. So a freshly registered module reports health `unknown` for a full
5515    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5516    // ready to answer.
5517    //
5518    // That is a property of the supervisor's schedule, not of any module: an
5519    // operator watching a restart sees `unknown` and cannot tell it from a module
5520    // that is slow to warm. Measured on two unrelated modules, both flipping to
5521    // `ok` between 22s and 32s after restart.
5522    //
5523    // Left as-is because spreading the first probe is what keeps a fleet-wide
5524    // restart from firing fourteen simultaneous probes into a cold machine. The
5525    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5526    // that thundering herd for a faster first reading.
5527    let jitter_span = (cadence_ms / 10).max(1);
5528    let hash = module_id.as_bytes().iter().fold(
5529        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5530        |acc, byte| {
5531            acc.wrapping_mul(1099511628211)
5532                .wrapping_add(u64::from(*byte))
5533        },
5534    );
5535    cadence + Duration::from_millis(hash % jitter_span)
5536}
5537
5538#[cfg(test)]
5539mod tests {
5540    use super::*;
5541
5542    #[test]
5543    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5544        let handle = SupervisorHandle::new();
5545        let module_id = "readded-tombstone";
5546        handle.record_rescan_removal(module_id);
5547        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5548
5549        handle.apply_identity_configuration(&ModuleSpec {
5550            module_id: module_id.to_string(),
5551            program: PathBuf::from("/test/module"),
5552            args: Vec::new(),
5553            env: Vec::new(),
5554            reserved: false,
5555            reserved_prefixes: Vec::new(),
5556            protocol: ModuleProtocol::Subc,
5557            overlap: Default::default(),
5558        });
5559
5560        assert!(
5561            handle.removal_tombstone_age_ms(module_id).is_none(),
5562            "a re-added module must not retain a stale removal tombstone"
5563        );
5564    }
5565
5566    /// What one module's owner looked like from the control plane at the
5567    /// instant after its first process was spawned.
5568    #[derive(Debug, PartialEq, Eq)]
5569    struct OwnerInSpawnWindow {
5570        module_id: String,
5571        configured: bool,
5572        on_roster: bool,
5573        admission_refusal: Option<&'static str>,
5574    }
5575
5576    /// A supervised module's process can connect, register, sync its scopes
5577    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5578    /// the module on the roster until afterward; configured supervision does
5579    /// so before spawning. In both cases the owner must
5580    /// already read as configured, so a scoped `route.open` against it is
5581    /// refused as retryable `scope_not_synced` and not as terminal
5582    /// `scope_not_live` ("will never sync").
5583    ///
5584    /// The hook runs just after each initial spawn attempt, before returning to
5585    /// the caller, so no race with a real child's registration is needed.
5586    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5587    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5588        use crate::scopes::ScopeTable;
5589        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5590
5591        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5592        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5593            module_id: module_id.to_string(),
5594            program,
5595            args: Vec::new(),
5596            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5597                .into_iter()
5598                .map(|key| (key.to_string(), dir.path().display().to_string()))
5599                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5600                .collect(),
5601            reserved: false,
5602            reserved_prefixes: Vec::new(),
5603            protocol: ModuleProtocol::Subc,
5604            overlap: Default::default(),
5605        };
5606        let live = super::terminal_history_tests::fake_aft_stub_path();
5607        let missing = dir.path().join("definitely-missing-module");
5608
5609        let handle = SupervisorHandle::new();
5610        let mut supervisor =
5611            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5612                .with_handle(handle.clone());
5613        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5614        let hook_handle = handle.clone();
5615        let hook_observed = Arc::clone(&observed);
5616        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5617            // Exactly what the control plane computes for a scoped route.open
5618            // naming this module as the owner of a scope it has not synced.
5619            let configured = hook_handle.is_configured(module_id);
5620            let selector = ScopeSelector::new(
5621                Principal::Reserved {
5622                    module_id: module_id.to_string(),
5623                },
5624                "s".to_string(),
5625                Some(1),
5626            );
5627            let carrier = Principal::Reserved {
5628                module_id: "carrier".to_string(),
5629            };
5630            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5631                .admit(&carrier, module_id, &selector, configured)
5632            {
5633                Ok(_) => None,
5634                Err(refusal) => Some(refusal.code),
5635            };
5636            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5637                module_id: module_id.to_string(),
5638                configured,
5639                on_roster: hook_handle.get(module_id).is_some(),
5640                admission_refusal,
5641            });
5642        })));
5643
5644        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5645        let configured = supervisor
5646            .supervise_configured(stub("configured", live.clone()), true)
5647            .unwrap();
5648        let with_health = supervisor
5649            .supervise_configured_with_health(
5650                stub("with-health", live.clone()),
5651                true,
5652                HealthConfig::default(),
5653                None,
5654                RestartPolicy::default(),
5655            )
5656            .unwrap();
5657        // The failed-spawn path still puts the module on the roster (as
5658        // failed), so it is configured throughout.
5659        let failed = supervisor
5660            .supervise_configured_with_health(
5661                stub("failed-spawn", missing.clone()),
5662                true,
5663                HealthConfig::default(),
5664                None,
5665                RestartPolicy::default(),
5666            )
5667            .unwrap();
5668        // A failed plain `spawn` puts nothing on the roster, so its mark is
5669        // taken back once the spawn has failed.
5670        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5671
5672        let expected = [
5673            "plain",
5674            "configured",
5675            "with-health",
5676            "failed-spawn",
5677            "spawn-error",
5678        ]
5679        .into_iter()
5680        .map(|module_id| OwnerInSpawnWindow {
5681            module_id: module_id.to_string(),
5682            configured: true,
5683            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5684            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5685        })
5686        .collect::<Vec<_>>();
5687        assert_eq!(*observed.lock().unwrap(), expected);
5688
5689        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5690            assert!(
5691                handle.get(module_id).is_some(),
5692                "{module_id} is on the roster"
5693            );
5694            assert!(
5695                handle.is_configured(module_id),
5696                "{module_id} stays configured"
5697            );
5698        }
5699        assert!(handle.get("spawn-error").is_none());
5700        assert!(
5701            !handle.is_configured("spawn-error"),
5702            "a plain spawn that failed must not leave its module marked configured"
5703        );
5704
5705        // Leaving the roster clears the mark with it.
5706        handle.retire("failed-spawn");
5707        assert!(!handle.is_configured("failed-spawn"));
5708
5709        for module in [plain, configured, with_health] {
5710            module.stop().await.unwrap();
5711        }
5712        drop(failed);
5713    }
5714
5715    /// A configured module reads as starting until the supervisor has admitted
5716    /// its first process (exec confirmed, identity durably recorded), even
5717    /// though that process is already running by then and can register over
5718    /// HELLO, which is what lists it in catalog.list. So a client that needs
5719    /// the supervisor state must wait for that state; seeing the module in
5720    /// the catalog does not mean supervisor.list already says running.
5721    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5722    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5723        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5724        let spec = ModuleSpec {
5725            module_id: "spawned-not-admitted".to_string(),
5726            program: super::terminal_history_tests::fake_aft_stub_path(),
5727            args: Vec::new(),
5728            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5729                .into_iter()
5730                .map(|key| (key.to_string(), dir.path().display().to_string()))
5731                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5732                .collect(),
5733            reserved: false,
5734            reserved_prefixes: Vec::new(),
5735            protocol: ModuleProtocol::Subc,
5736            overlap: Default::default(),
5737        };
5738        let handle = SupervisorHandle::new();
5739        let mut supervisor =
5740            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5741                .with_handle(handle.clone());
5742        let observed = Arc::new(Mutex::new(None));
5743        let hook_handle = handle.clone();
5744        let hook_observed = Arc::clone(&observed);
5745        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5746            let state = hook_handle
5747                .get(module_id)
5748                .expect("a configured module is on the roster before it is spawned")
5749                .state()
5750                .unwrap();
5751            *hook_observed.lock().unwrap() = Some(state);
5752        })));
5753
5754        let module = supervisor.supervise_configured(spec, true).unwrap();
5755        assert_eq!(
5756            *observed.lock().unwrap(),
5757            Some(ModuleState::Starting),
5758            "between the spawn and its admission the supervisor still reports starting"
5759        );
5760        assert_eq!(module.state().unwrap(), ModuleState::Running);
5761        module.stop().await.unwrap();
5762    }
5763
5764    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5765        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5766        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5767            snapshot.process_alive = true;
5768            snapshot.pid = Some(41);
5769            snapshot.spawned_at_ms = Some(42);
5770            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5771            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5772            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5773                device: 43,
5774                inode: 44,
5775                #[cfg(windows)]
5776                windows_image: None,
5777            });
5778        })
5779        .unwrap();
5780        snapshot
5781    }
5782
5783    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5784        let snapshot = lock_snapshot(snapshot).unwrap();
5785        assert!(!snapshot.process_alive);
5786        assert_eq!(snapshot.pid, None);
5787        assert_eq!(snapshot.spawned_at_ms, None);
5788        assert_eq!(snapshot.spawned_from, None);
5789        assert_eq!(snapshot.launch_spec_digest, None);
5790        assert_eq!(snapshot.spawned_file_identity, None);
5791    }
5792
5793    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5794    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5795        let supervisor =
5796            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5797        let mut runtime = supervisor.runtime_config();
5798        runtime.test_seed_stale_facts_before_enable_spawn = true;
5799        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5800        let mut child = None;
5801        let spec = ModuleSpec {
5802            module_id: "failed-enable-clears-facts".to_string(),
5803            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5804            args: Vec::new(),
5805            env: Vec::new(),
5806            reserved: false,
5807            reserved_prefixes: Vec::new(),
5808            protocol: ModuleProtocol::Subc,
5809            overlap: Default::default(),
5810        };
5811
5812        let result = set_child_enabled(
5813            &spec,
5814            &runtime,
5815            &supervisor.registry,
5816            &supervisor.process_liveness,
5817            &snapshot,
5818            &mut child,
5819            true,
5820        )
5821        .await;
5822
5823        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5824        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5825        assert_snapshot_process_facts_cleared(&snapshot);
5826    }
5827
5828    #[tokio::test]
5829    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5830        let supervisor =
5831            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5832        let runtime = supervisor.runtime_config();
5833        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5834            ModuleState::Restarting,
5835            true,
5836        )));
5837        let spec = ModuleSpec {
5838            module_id: "start-stranded-restarting".to_string(),
5839            program: super::terminal_history_tests::fake_aft_stub_path(),
5840            args: Vec::new(),
5841            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5842            reserved: false,
5843            reserved_prefixes: Vec::new(),
5844            protocol: ModuleProtocol::None,
5845            overlap: Default::default(),
5846        };
5847        let mut child = None;
5848        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5849        assert!(!super::set_child_enabled(
5850            &spec,
5851            &runtime,
5852            &Registry::default(),
5853            &supervisor.process_liveness,
5854            &snapshot,
5855            &mut child,
5856            true
5857        )
5858        .await
5859        .unwrap());
5860        assert!(child.is_none());
5861        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5862        assert!(super::set_child_enabled(
5863            &spec,
5864            &runtime,
5865            &Registry::default(),
5866            &supervisor.process_liveness,
5867            &snapshot,
5868            &mut child,
5869            true
5870        )
5871        .await
5872        .unwrap());
5873        assert_eq!(
5874            lock_snapshot(&snapshot).unwrap().state,
5875            ModuleState::Running
5876        );
5877        let mut child = child.unwrap();
5878        child.start_kill().unwrap();
5879        child.wait().await.unwrap();
5880    }
5881
5882    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5883    async fn failed_reload_spawn_clears_current_process_facts() {
5884        let supervisor =
5885            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5886        let mut runtime = supervisor.runtime_config();
5887        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5888        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5889        let mut child = None;
5890        let spec = ModuleSpec {
5891            module_id: "failed-reload-clears-facts".to_string(),
5892            program: PathBuf::from("/unused/failed-reload-module"),
5893            args: Vec::new(),
5894            env: Vec::new(),
5895            reserved: false,
5896            reserved_prefixes: Vec::new(),
5897            protocol: ModuleProtocol::Subc,
5898            overlap: Default::default(),
5899        };
5900
5901        let result = handle_reload_spawn_failure(
5902            &spec,
5903            &runtime,
5904            &supervisor.process_liveness,
5905            &snapshot,
5906            &mut child,
5907            "forced reload spawn failure".to_string(),
5908        )
5909        .await;
5910
5911        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5912        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5913        assert_snapshot_process_facts_cleared(&snapshot);
5914    }
5915
5916    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5917    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5918        let supervisor =
5919            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5920        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5921        let module = supervisor.supervised_module(
5922            ModuleSpec {
5923                module_id: "drop-clears-facts".to_string(),
5924                program: PathBuf::from("/unused/drop-module"),
5925                args: Vec::new(),
5926                env: Vec::new(),
5927                reserved: false,
5928                reserved_prefixes: Vec::new(),
5929                protocol: ModuleProtocol::Subc,
5930                overlap: Default::default(),
5931            },
5932            supervisor.runtime_config(),
5933            Arc::clone(&snapshot),
5934            None,
5935        );
5936        assert!(!module
5937            .inner
5938            .monitor
5939            .lock()
5940            .unwrap()
5941            .as_ref()
5942            .unwrap()
5943            .is_finished());
5944
5945        drop(module);
5946
5947        assert_eq!(
5948            lock_snapshot(&snapshot).unwrap().state,
5949            ModuleState::Stopped
5950        );
5951        assert_snapshot_process_facts_cleared(&snapshot);
5952    }
5953
5954    #[cfg(unix)]
5955    #[tokio::test]
5956    async fn rescan_preserves_running_protocol_until_respawn() {
5957        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5958        let initial = ModuleSpec {
5959            module_id: "rescan-protocol".into(),
5960            program: PathBuf::from("/bin/sleep"),
5961            args: vec!["60".into()],
5962            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5963                .into_iter()
5964                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5965                .collect(),
5966            reserved: false,
5967            reserved_prefixes: vec![],
5968            protocol: ModuleProtocol::None,
5969            overlap: Default::default(),
5970        };
5971        let supervisor =
5972            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5973        let module = supervisor.spawn(initial.clone()).unwrap();
5974        assert!(module.status().unwrap().live);
5975        let mut next = initial;
5976        next.protocol = ModuleProtocol::Subc;
5977        module
5978            .update_configuration(next.clone(), HealthConfig::default(), None)
5979            .await
5980            .unwrap();
5981        assert!(
5982            module.status().unwrap().live,
5983            "rescan must not require HELLO from the old non-wire process"
5984        );
5985        let runtime = supervisor.runtime_config();
5986        let action = on_child_exit(
5987            &next,
5988            RestartPolicy::default(),
5989            &supervisor.registry,
5990            &module.inner.snapshot,
5991            &runtime.terminal_ring,
5992            &runtime.spawn_events,
5993            &runtime.child_roster,
5994            ExitReport {
5995                kind: ExitKind::Clean,
5996                code: Some(0),
5997                signal: None,
5998                at_ms: unix_ms_now(),
5999            },
6000        )
6001        .await;
6002        assert!(
6003            matches!(action, NextAction::Restart { .. }),
6004            "the old non-wire process's clean exit must restart"
6005        );
6006        module.drain().await.unwrap();
6007    }
6008
6009    #[cfg(unix)]
6010    #[tokio::test]
6011    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6012        use std::os::unix::fs::PermissionsExt;
6013        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6014        let script = dir.join("module.sh");
6015        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6016        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6017        let record_path = dir.join("live-children.json");
6018        let supervisor =
6019            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6020                .with_live_children_record(&record_path);
6021        for (program, args) in [
6022            (PathBuf::from("sleep"), vec!["60".into()]),
6023            (script, vec![]),
6024        ] {
6025            let spec = ModuleSpec {
6026                module_id: "image-identity".into(),
6027                program: program.clone(),
6028                args,
6029                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6030                    .into_iter()
6031                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6032                    .collect(),
6033                reserved: false,
6034                reserved_prefixes: vec![],
6035                protocol: ModuleProtocol::None,
6036                overlap: Default::default(),
6037            };
6038            let module = supervisor.spawn(spec).unwrap();
6039            #[cfg(target_os = "macos")]
6040            {
6041                // SETEXEC confirmation is asynchronous; the orphan record must
6042                // identify the final image, never the intermediate trampoline.
6043                let deadline = Instant::now() + Duration::from_secs(5);
6044                while crate::live_children::read_record(&record_path)
6045                    .unwrap()
6046                    .iter()
6047                    .all(|entry| entry.executable.is_none())
6048                {
6049                    assert!(Instant::now() < deadline, "module image was not confirmed");
6050                    tokio::time::sleep(Duration::from_millis(5)).await;
6051                }
6052            }
6053            let entry = crate::live_children::read_record(&record_path)
6054                .unwrap()
6055                .pop()
6056                .unwrap();
6057            let observed = subc_os::Process::open(entry.pid)
6058                .unwrap()
6059                .unwrap()
6060                .observe()
6061                .unwrap();
6062            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6063            module.drain().await.unwrap();
6064            assert_eq!(
6065                verdict,
6066                crate::live_children::IdentityVerdict::Matches,
6067                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6068            );
6069        }
6070    }
6071
6072    #[cfg(unix)]
6073    fn http_fixture(
6074        dir: &std::path::Path,
6075        url: &str,
6076        threshold: u32,
6077    ) -> crate::daemon_config::ConfiguredModule {
6078        let path = dir.join("subc.jsonc");
6079        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6080            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6081            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6082            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6083            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6084        }}}).to_string()).unwrap();
6085        crate::daemon_config::load(&path)
6086            .unwrap()
6087            .unwrap()
6088            .modules
6089            .pop()
6090            .unwrap()
6091    }
6092
6093    #[cfg(unix)]
6094    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6095        timeout(Duration::from_secs(5), async {
6096            loop {
6097                if module.status().unwrap().health.status == status {
6098                    break;
6099                }
6100                sleep(Duration::from_millis(5)).await;
6101            }
6102        })
6103        .await
6104        .unwrap_or_else(|_| {
6105            panic!(
6106                "expected {status:?}, got {:?}",
6107                module.status().unwrap().health
6108            )
6109        });
6110    }
6111
6112    #[cfg(unix)]
6113    #[tokio::test]
6114    async fn http_health_status_flips_ok_failing_ok() {
6115        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6116        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6117        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6118        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6119        let serving_status = status.clone();
6120        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6121        let server = tokio::spawn(async move {
6122            loop {
6123                let (mut stream, _) = listener.accept().await.unwrap();
6124                let mut request = [0u8; 2048];
6125                let count = stream.read(&mut request).await.unwrap();
6126                assert!(count > 0, "a probe must send an HTTP request");
6127                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6128                let body = if code == 200 {
6129                    "ready"
6130                } else {
6131                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6132                };
6133                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6134                let _ = stream.write_all(response.as_bytes()).await;
6135            }
6136        });
6137        let configured = http_fixture(&dir, &url, 1000);
6138        let module =
6139            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6140                .supervise_configured_with_health(
6141                    configured.module_spec(),
6142                    true,
6143                    configured.health,
6144                    configured.drain_timeout_ms,
6145                    configured.restart,
6146                )
6147                .unwrap();
6148        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6149        status.store(503, std::sync::atomic::Ordering::SeqCst);
6150        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6151        assert!(module
6152            .status()
6153            .unwrap()
6154            .health
6155            .detail
6156            .unwrap()
6157            .contains("scratch failure"));
6158        status.store(200, std::sync::atomic::Ordering::SeqCst);
6159        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6160        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6161        let before = module.status().unwrap();
6162        let (spec, mut health) = module.configuration().unwrap();
6163        health.http = None;
6164        module
6165            .update_configuration(spec.clone(), health.clone(), Some(10))
6166            .await
6167            .unwrap();
6168        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6169        health.http = Some(url);
6170        module
6171            .update_configuration(spec, health, Some(10))
6172            .await
6173            .unwrap();
6174        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6175        assert_eq!(
6176            module.status().unwrap().pid,
6177            before.pid,
6178            "changing a probe must apply live, not restart its process"
6179        );
6180        let (spec, mut health) = module.configuration().unwrap();
6181        health.failure_threshold = 2;
6182        module
6183            .update_configuration(spec, health, Some(10))
6184            .await
6185            .unwrap();
6186        status.store(503, std::sync::atomic::Ordering::SeqCst);
6187        timeout(Duration::from_secs(5), async {
6188            while module.status().unwrap().spawn_generation == before.spawn_generation {
6189                sleep(Duration::from_millis(5)).await;
6190            }
6191        })
6192        .await
6193        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6194        module.drain().await.unwrap();
6195        server.abort();
6196    }
6197
6198    #[cfg(unix)]
6199    #[tokio::test]
6200    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6201        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6202        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6203        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6204        drop(unused);
6205        let configured = http_fixture(&dir, &url, 2);
6206        let module =
6207            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6208                .supervise_configured_with_health(
6209                    configured.module_spec(),
6210                    true,
6211                    configured.health,
6212                    configured.drain_timeout_ms,
6213                    configured.restart,
6214                )
6215                .unwrap();
6216        let before = module.status().unwrap().spawn_generation;
6217        timeout(Duration::from_secs(5), async {
6218            loop {
6219                let status = module.status().unwrap();
6220                if status.spawn_generation > before {
6221                    assert!(status.lifetime_restarts > 0);
6222                    break;
6223                }
6224                sleep(Duration::from_millis(5)).await;
6225            }
6226        })
6227        .await
6228        .expect("sustained HTTP refusal must trigger the health restart policy");
6229        module.drain().await.unwrap();
6230    }
6231
6232    #[cfg(unix)]
6233    #[tokio::test]
6234    async fn http_health_timeout_honours_deadline() {
6235        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6236        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6237        let server = tokio::spawn(async move {
6238            let _held = listener.accept().await.unwrap();
6239            std::future::pending::<()>().await;
6240        });
6241        let error = timeout(
6242            Duration::from_secs(1),
6243            probe_http_health(&url, Duration::from_millis(10)),
6244        )
6245        .await
6246        .expect("the probe must enforce its own deadline")
6247        .unwrap_err();
6248        server.abort();
6249        assert!(error.to_string().contains("timed out"));
6250    }
6251
6252    #[cfg(unix)]
6253    #[tokio::test]
6254    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6255        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6256        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6257        let url = format!(
6258            "http://localhost:{}/healthz",
6259            listener.local_addr().unwrap().port()
6260        );
6261        let server = tokio::spawn(async move {
6262            let (mut stream, _) = listener.accept().await.unwrap();
6263            let mut request = [0u8; 2048];
6264            assert!(stream.read(&mut request).await.unwrap() > 0);
6265            stream
6266                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6267                .await
6268                .unwrap();
6269        });
6270        // The deadline only bounds a hang. A probe that never tried the IPv6
6271        // address would be refused on 127.0.0.1 and fail at once, so a longer
6272        // deadline does not weaken the assertion; one second timed out under a
6273        // loaded parallel test run.
6274        assert_eq!(
6275            probe_http_health(&url, Duration::from_secs(10))
6276                .await
6277                .unwrap()
6278                .status,
6279            HealthStatus::Ok
6280        );
6281        server.await.unwrap();
6282    }
6283
6284    #[cfg(unix)]
6285    #[tokio::test]
6286    async fn http_health_timeout_keeps_partial_status_and_body() {
6287        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6288        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6289        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6290        let server = tokio::spawn(async move {
6291            let (mut stream, _) = listener.accept().await.unwrap();
6292            let mut request = [0u8; 2048];
6293            assert!(stream.read(&mut request).await.unwrap() > 0);
6294            stream
6295                .write_all(
6296                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6297                )
6298                .await
6299                .unwrap();
6300            std::future::pending::<()>().await;
6301        });
6302        let error = probe_http_health(&url, Duration::from_secs(1))
6303            .await
6304            .unwrap_err()
6305            .to_string();
6306        server.abort();
6307        assert!(
6308            error.contains("timed out")
6309                && error.contains("503 Unavailable")
6310                && error.contains("partial diagnostic"),
6311            "{error}"
6312        );
6313    }
6314
6315    #[cfg(unix)]
6316    #[tokio::test]
6317    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6318        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6319        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6320        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6321        let server = tokio::spawn(async move {
6322            let (mut stream, _) = listener.accept().await.unwrap();
6323            let mut request = [0u8; 2048];
6324            assert!(stream.read(&mut request).await.unwrap() > 0);
6325            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6326            let response = format!(
6327                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6328                body.len()
6329            );
6330            stream.write_all(response.as_bytes()).await.unwrap();
6331        });
6332        let error = probe_http_health(&url, Duration::from_secs(1))
6333            .await
6334            .unwrap_err()
6335            .to_string();
6336        server.await.unwrap();
6337        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6338        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6339        assert!(!error.contains("not-in-diagnostic"));
6340    }
6341
6342    #[cfg(unix)]
6343    #[tokio::test]
6344    async fn http_health_real_nats_server_monitoring() {
6345        if std::process::Command::new("nats-server")
6346            .arg("--version")
6347            .env("XDG_DATA_HOME", std::env::temp_dir())
6348            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6349            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6350            .output()
6351            .is_err()
6352        {
6353            eprintln!(
6354                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6355            );
6356            return;
6357        }
6358        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6359        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6360        let port = monitor.local_addr().unwrap().port();
6361        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6362        let client_port = client.local_addr().unwrap().port();
6363        let config = dir.join("server.conf");
6364        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6365        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6366        configured.program = PathBuf::from("nats-server");
6367        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6368        drop(monitor);
6369        drop(client);
6370        let module =
6371            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6372                .supervise_configured_with_health(
6373                    configured.module_spec(),
6374                    true,
6375                    configured.health,
6376                    configured.drain_timeout_ms,
6377                    configured.restart,
6378                )
6379                .unwrap();
6380        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6381        module.drain().await.unwrap();
6382    }
6383
6384    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6385    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6386        let supervisor =
6387            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6388        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6389        let initial = ModuleSpec {
6390            module_id: "rescan-preserves-spawn-facts".to_string(),
6391            program: PathBuf::from("/spawned/module"),
6392            args: Vec::new(),
6393            env: Vec::new(),
6394            reserved: false,
6395            reserved_prefixes: Vec::new(),
6396            protocol: ModuleProtocol::Subc,
6397            overlap: Default::default(),
6398        };
6399        let module = supervisor.supervised_module(
6400            initial.clone(),
6401            supervisor.runtime_config(),
6402            snapshot,
6403            None,
6404        );
6405        let before = module.status().unwrap();
6406        let mut replacement = initial;
6407        replacement.program = PathBuf::from("/rescanned/replacement-module");
6408
6409        module
6410            .update_configuration(replacement, HealthConfig::default(), None)
6411            .await
6412            .unwrap();
6413
6414        let after = module.status().unwrap();
6415        assert_eq!(after.pid, before.pid);
6416        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6417        assert_eq!(after.spawned_from, before.spawned_from);
6418        drop(module);
6419    }
6420}
6421
6422fn unix_ms_now() -> u64 {
6423    SystemTime::now()
6424        .duration_since(UNIX_EPOCH)
6425        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6426        .unwrap_or(0)
6427}
6428
6429async fn supervise_loop(
6430    mut spec: ModuleSpec,
6431    mut runtime: SupervisorRuntimeConfig,
6432    registry: Arc<Registry>,
6433    process_liveness: Arc<SupervisorProcessLiveness>,
6434    snapshot: SharedSnapshot,
6435    mut child: Option<SupervisedChild>,
6436    mut commands: mpsc::Receiver<SupervisorCommand>,
6437) {
6438    let mut health_probe = HealthProbeRuntime::default();
6439    // Registry writes (including embedded callers) notify this module only.
6440    // Subscribe before the first refresh; watch retains changes that arrive
6441    // while commands or probes are running. No polling fallback is needed.
6442    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6443        Ok(changes) => Some(changes),
6444        Err(err) => {
6445            // A poisoned registry cannot accept further writes, so it cannot
6446            // recover via a timer. Keep exit and command handling alive.
6447            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6448            None
6449        }
6450    };
6451    // All restart backoffs run here, including health and operator requests.
6452    // While one is pending the loop serves commands, so disable or drain can
6453    // cancel the replacement without spawning a process just to stop it.
6454    let mut pending_respawn: Option<PendingRespawn> = None;
6455    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6456    // before anything else so a stop that interrupted a swap runs at once.
6457    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6458    let mut previous_instance_recheck = tokio::time::interval_at(
6459        Instant::now() + Duration::from_secs(5),
6460        Duration::from_secs(5),
6461    );
6462    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6463    loop {
6464        #[cfg(test)]
6465        {
6466            lock_snapshot(&snapshot).unwrap().actor_select = None;
6467        }
6468        #[cfg(target_os = "macos")]
6469        if let Some(active) = child.as_mut() {
6470            active.confirm_privacy_exec().await;
6471        }
6472        if let Some(scheduled) = runtime
6473            .scheduled_respawn
6474            .lock()
6475            .unwrap_or_else(|p| p.into_inner())
6476            .take()
6477        {
6478            pending_respawn = Some(scheduled);
6479        }
6480        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6481            pending_respawn = None;
6482            cancel_deferred_reload(
6483                &runtime,
6484                &spec.module_id,
6485                "respawn cancelled by a supervisor command",
6486            );
6487        }
6488        if child.is_none()
6489            && pending_respawn.is_none()
6490            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6491        {
6492            cancel_deferred_reload(
6493                &runtime,
6494                &spec.module_id,
6495                "respawn cancelled before a replacement was spawned",
6496            );
6497            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6498                state.respawn_pending = false;
6499                state.coalesced_restart_pending = false;
6500                if matches!(
6501                    state.state,
6502                    ModuleState::Restarting
6503                        | ModuleState::Starting
6504                        | ModuleState::Draining
6505                        | ModuleState::Unresponsive
6506                ) {
6507                    error!(module_id = %spec.module_id, state = ?state.state,
6508                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6509                    state.state = ModuleState::Failed;
6510                    clear_current_process_facts(state);
6511                }
6512            });
6513        }
6514        if let Some(command) = requeued.pop_front() {
6515            if !handle_supervisor_command(
6516                command,
6517                &mut spec,
6518                &mut runtime,
6519                &registry,
6520                &process_liveness,
6521                &snapshot,
6522                &mut child,
6523                &mut commands,
6524                &mut requeued,
6525            )
6526            .await
6527            {
6528                return;
6529            }
6530            if child.is_some() || !respawn_still_pending(&snapshot) {
6531                pending_respawn = None;
6532                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6533                    state.respawn_pending = false
6534                });
6535            }
6536            continue;
6537        }
6538        if child.is_some() {
6539            #[cfg(test)]
6540            {
6541                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6542            }
6543            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6544            let wake_after = health_probe.wake_after();
6545            let probe_sleep = async move {
6546                match wake_after {
6547                    Some(delay) => sleep(delay).await,
6548                    None => std::future::pending().await,
6549                }
6550            };
6551            tokio::pin!(probe_sleep);
6552            // A `protocol: "none"` child (a plain process that never registers
6553            // over the subc wire, such as nats-server) with no HTTP health
6554            // check has nothing to probe, so it waits only for its exit or a
6555            // supervisor command. A rescan that adds an HTTP check is a command.
6556            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6557            #[cfg(test)]
6558            {
6559                let registration_pending = wire_child
6560                    && registration_changes
6561                        .as_ref()
6562                        .is_some_and(|changes| changes.has_changed().unwrap());
6563                let mut state = lock_snapshot(&snapshot).unwrap();
6564                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6565                    generation: state.spawn_generation,
6566                    turn: state.actor_turns,
6567                    registered_connection: health_probe.registered_connection,
6568                    next_probe_at: health_probe.next_probe_at,
6569                    wake_after,
6570                });
6571            }
6572            let active_child = child.as_mut().expect("child checked above");
6573            tokio::select! {
6574                wait_result = active_child.wait() => {
6575                    #[cfg(test)]
6576                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6577                    // Every arm below that gives up on the CHILD must keep the
6578                    // supervision task itself alive (child = None, loop
6579                    // continues into command-serving mode). Returning here
6580                    // closes the command channel, which makes the module
6581                    // permanently unrestartable in-band: a clean child exit
6582                    // of an enabled module once wedged the fleet this way
6583                    // ('supervisor command channel is closed') and required a
6584                    // full daemon restart to recover.
6585                    let exit_report = match wait_result {
6586                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6587                        Err(err) => {
6588                            active_child.drain_stderr(&spec.module_id).await;
6589                            fail_after_wait_error(
6590                                &spec.module_id,
6591                                &snapshot,
6592                                &runtime.terminal_ring,
6593                                &runtime.spawn_events,
6594                                format!("failed to wait for supervised module: {err}"),
6595                            );
6596                            untrack_if_registration_released(
6597                                &process_liveness,
6598                                &registry,
6599                                &spec.module_id,
6600                                &snapshot,
6601                            );
6602                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6603                            child = None;
6604                            continue;
6605                        }
6606                    };
6607                    active_child.drain_stderr(&spec.module_id).await;
6608
6609                    let next = on_child_exit(
6610                        &spec,
6611                        runtime.restart_policy,
6612                        &registry,
6613                        &snapshot,
6614                        &runtime.terminal_ring,
6615                        &runtime.spawn_events,
6616                        &runtime.child_roster,
6617                        exit_report,
6618                    ).await;
6619                    runtime.start_requester = None;
6620                    runtime.pending_activity_restart = false;
6621                    // The exit is recorded, so a daemon shutdown may stop
6622                    // waiting for this child (see `SupervisedChild::wait`).
6623                    active_child.release_roster();
6624                    match next {
6625                        NextAction::Stop { registration_released } => {
6626                            if registration_released {
6627                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6628                            }
6629                            child = None;
6630                        }
6631                        NextAction::Restart { schedule } => {
6632                            let delay = schedule.map_or(
6633                                runtime.restart_policy.delay_for_restart(0),
6634                                |schedule| schedule.delay,
6635                            );
6636                            if let Some(schedule) = schedule {
6637                                log_crash_respawn(&spec.module_id, schedule);
6638                            }
6639                            // The exited child is fully recorded at this point,
6640                            // so release it and count the backoff down in the
6641                            // command-serving branch below rather than sleeping
6642                            // here: commands cannot be received from inside this
6643                            // select arm, and an operator disable or drain that
6644                            // arrives during the backoff must cancel the pending
6645                            // respawn instead of waiting for it to spawn first.
6646                            child = None;
6647                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6648                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6649                        }
6650                    }
6651                }
6652                command = commands.recv() => {
6653                    #[cfg(test)]
6654                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6655                    let Some(command) = command else {
6656                        return;
6657                    };
6658                    if !handle_supervisor_command(
6659                        command,
6660                        &mut spec,
6661                        &mut runtime,
6662                        &registry,
6663                        &process_liveness,
6664                        &snapshot,
6665                        &mut child,
6666                        &mut commands,
6667                        &mut requeued,
6668                    ).await {
6669                        return;
6670                    }
6671                }
6672                _ = async {
6673                    match registration_changes.as_mut() {
6674                        Some(changes) => { let _ = changes.changed().await; }
6675                        None => std::future::pending().await,
6676                    }
6677                }, if wire_child => {
6678                    #[cfg(test)]
6679                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6680                }
6681                _ = &mut probe_sleep => {
6682                    #[cfg(test)]
6683                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6684                    if health_probe.due() {
6685                        run_health_probe_cycle(
6686                            &spec,
6687                            &runtime,
6688                            &registry,
6689                            &process_liveness,
6690                            &snapshot,
6691                            &mut child,
6692                        ).await;
6693                        if child.is_none() {
6694                            runtime.start_requester = None;
6695                            runtime.pending_activity_restart = false;
6696                        }
6697                        if child.is_some() {
6698                            health_probe.schedule_next(&spec, runtime.health.cadence);
6699                        }
6700                    }
6701                }
6702            }
6703        } else if let Some(pending) = pending_respawn {
6704            tokio::select! {
6705                _ = sleep_until(pending.deadline) => {
6706                    pending_respawn = None;
6707                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6708                    // A command handled below while the backoff elapsed may
6709                    // have stopped the module; never respawn past an operator's
6710                    // disable or drain.
6711                    if !respawn_still_pending(&snapshot) {
6712                        continue;
6713                    }
6714                    // The daemon began shutting down during the backoff: the
6715                    // spawn would be refused anyway, and refusing it here
6716                    // leaves the module stopped instead of reporting a
6717                    // failed restart.
6718                    if runtime.child_roster.is_closed() {
6719                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6720                            state.state = ModuleState::Stopped;
6721                        });
6722                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6723                        continue;
6724                    }
6725                    if let Err(err) = release_dead_registration(
6726                        &registry,
6727                        runtime.forwarding.as_deref(),
6728                        &snapshot,
6729                        &spec.module_id,
6730                    ).await {
6731                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6732                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6733                        continue;
6734                    }
6735
6736                    if matches!(pending.kind, RespawnKind::Reload) {
6737                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6738                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6739                        if result.is_ok() && runtime.pending_activity_restart {
6740                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6741                        }
6742                        runtime.pending_activity_restart = false;
6743                        if let Some(reply) = reply { let _ = reply.send(result); }
6744                        continue;
6745                    }
6746                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6747                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6748                        Ok(next_child) => {
6749                            child = Some(next_child);
6750                            if runtime.pending_activity_restart {
6751                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6752                                runtime.pending_activity_restart = false;
6753                            }
6754                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6755                        }
6756                        Err(err) => {
6757                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6758                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6759                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6760                        }
6761                    }
6762                }
6763                command = commands.recv() => {
6764                    let Some(command) = command else {
6765                        return;
6766                    };
6767                    if !handle_supervisor_command(
6768                        command,
6769                        &mut spec,
6770                        &mut runtime,
6771                        &registry,
6772                        &process_liveness,
6773                        &snapshot,
6774                        &mut child,
6775                        &mut commands,
6776                        &mut requeued,
6777                    ).await {
6778                        return;
6779                    }
6780                    // Reconcile the pending respawn with what the command did:
6781                    // a start may already have spawned a fresh child,
6782                    // while a disable or drain moved the snapshot out of the
6783                    // state the respawn was counting down from.
6784                    if child.is_some() || !respawn_still_pending(&snapshot) {
6785                        pending_respawn = None;
6786                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6787                    }
6788                }
6789            }
6790        } else {
6791            let command = if lock_snapshot(&snapshot)
6792                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6793            {
6794                tokio::select! {
6795                    command = commands.recv() => command,
6796                    _ = previous_instance_recheck.tick() => {
6797                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6798                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6799                            if enabled && !runtime.child_roster.is_closed() {
6800                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6801                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6802                                    Ok(next_child) => child = Some(next_child),
6803                                    Err(error) => {
6804                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6805                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6806                                    }
6807                                }
6808                            } else {
6809                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6810                                    state.waiting_for_previous_pid = None;
6811                                    state.waiting_for_previous_detail = None;
6812                                });
6813                            }
6814                        }
6815                        continue;
6816                    }
6817                }
6818            } else {
6819                commands.recv().await
6820            };
6821            let Some(command) = command else {
6822                return;
6823            };
6824            if !handle_supervisor_command(
6825                command,
6826                &mut spec,
6827                &mut runtime,
6828                &registry,
6829                &process_liveness,
6830                &snapshot,
6831                &mut child,
6832                &mut commands,
6833                &mut requeued,
6834            )
6835            .await
6836            {
6837                return;
6838            }
6839        }
6840    }
6841}
6842
6843fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6844    info!(
6845        module_id,
6846        restart_in_window = schedule.restart_in_window,
6847        delay_ms = schedule.delay.as_millis() as u64,
6848        "respawning after crash"
6849    );
6850}
6851
6852/// Whether the respawn a backoff was counting down to is still wanted. A
6853/// disable or drain handled while the backoff elapsed moves the snapshot out
6854/// of `Restarting`, and the operator's stop must win over the pending respawn,
6855/// so every sleep-then-spawn path re-validates against the live snapshot
6856/// instead of assuming the state it left behind still holds.
6857fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6858    matches!(
6859        lock_snapshot(snapshot),
6860        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6861    )
6862}
6863
6864enum NextAction {
6865    Stop {
6866        registration_released: bool,
6867    },
6868    Restart {
6869        schedule: Option<CrashRestartSchedule>,
6870    },
6871}
6872
6873#[allow(clippy::too_many_arguments)]
6874async fn handle_supervisor_command(
6875    command: SupervisorCommand,
6876    spec: &mut ModuleSpec,
6877    runtime: &mut SupervisorRuntimeConfig,
6878    registry: &Arc<Registry>,
6879    process_liveness: &SupervisorProcessLiveness,
6880    snapshot: &SharedSnapshot,
6881    child: &mut Option<SupervisedChild>,
6882    commands: &mut mpsc::Receiver<SupervisorCommand>,
6883    requeued: &mut VecDeque<SupervisorCommand>,
6884) -> bool {
6885    let command =
6886        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6887            match command {
6888                SupervisorCommand::Restart { reply, .. }
6889                | SupervisorCommand::Reload { reply, .. }
6890                | SupervisorCommand::Swap { reply, .. } => {
6891                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6892                    return true;
6893                }
6894                SupervisorCommand::SetEnabled {
6895                    enabled: true,
6896                    reply,
6897                    ..
6898                } => {
6899                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6900                    return true;
6901                }
6902                other => other,
6903            }
6904        } else {
6905            command
6906        };
6907    match command {
6908        SupervisorCommand::Drain { requester, reply } => {
6909            // A plain stop runs no forwarding drain, so nothing reaches the
6910            // module over its connection before the wait: ask by signal.
6911            let result = drain_optional_child(
6912                &spec.module_id,
6913                spec.protocol,
6914                StopNotice::NotSent,
6915                registry,
6916                runtime.forwarding.as_deref(),
6917                snapshot,
6918                &runtime.terminal_ring,
6919                &runtime.spawn_events,
6920                child,
6921                runtime.drain_timeout,
6922                ModuleState::Stopped,
6923                None,
6924            )
6925            .await;
6926            let registration_released = result.is_ok();
6927            if registration_released {
6928                runtime.record_activity_transition(
6929                    "subc.module_stopped",
6930                    &spec.module_id,
6931                    requester,
6932                );
6933            }
6934            let _ = reply.send(result);
6935            if registration_released {
6936                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6937            }
6938            false
6939        }
6940        SupervisorCommand::Retire { requester, reply } => {
6941            let result = async {
6942                let stop_notice = begin_forwarding_drain_if_configured(
6943                    spec,
6944                    runtime,
6945                    registry,
6946                    snapshot,
6947                    None,
6948                    RouteCloseReason::Disable,
6949                )
6950                .await?;
6951                drain_optional_child(
6952                    &spec.module_id,
6953                    spec.protocol,
6954                    stop_notice,
6955                    registry,
6956                    runtime.forwarding.as_deref(),
6957                    snapshot,
6958                    &runtime.terminal_ring,
6959                    &runtime.spawn_events,
6960                    child,
6961                    runtime.drain_timeout,
6962                    ModuleState::Stopped,
6963                    None,
6964                )
6965                .await
6966            }
6967            .await;
6968            let registration_released = result.is_ok();
6969            if registration_released {
6970                runtime.record_activity_transition(
6971                    "subc.module_stopped",
6972                    &spec.module_id,
6973                    requester,
6974                );
6975            }
6976            let _ = reply.send(result);
6977            if registration_released {
6978                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6979            }
6980            false
6981        }
6982        SupervisorCommand::Restart {
6983            requester,
6984            drain_timeout_ms,
6985            received_at_generation,
6986            queued_at,
6987            reply,
6988        } => {
6989            // Without this line a restart that waited in the queue (behind a
6990            // health probe cycle or another command) was invisible: the log
6991            // showed only the drain timing out, minutes after the operator's call.
6992            info!(
6993                module_id = %spec.module_id,
6994                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6995                "restart command dequeued"
6996            );
6997            // ACK AT INITIATION, not completion. The blocking form deadlocked any
6998            // caller whose own request lane rides the module being restarted: the
6999            // caller's in-flight request keeps the drain from quiescing, the drain
7000            // keeps the restart from completing, and the completion keeps the reply
7001            // from releasing the caller — so the drain always timed out and cut the
7002            // initiator with a GOODBYE, even on a healthy module. Replying once the
7003            // restart is validated lets a self-lane caller settle, which is exactly
7004            // what makes the drain succeed. Completion is observable via
7005            // supervisor.list / module status; a post-ack failure lands the module
7006            // in a visible terminal state below rather than in a reply nobody can
7007            // receive.
7008            let validation = match lock_snapshot(snapshot) {
7009                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7010                    module_id: spec.module_id.clone(),
7011                }),
7012                Ok(_) => Ok(()),
7013                Err(err) => Err(err),
7014            };
7015            let initiated = validation.is_ok();
7016            let _ = reply.send(validation);
7017            // A restart asks for a fresh process. Commands run one at a time,
7018            // so a restart queued behind another restart (two operator calls
7019            // in quick succession) is dequeued the moment the first one has
7020            // spawned its replacement -- before that process has sent HELLO.
7021            // Running it would drain and kill the process the first restart
7022            // just produced, which is the opposite of what both callers asked
7023            // for. If a process spawned after this request was received is
7024            // still supervised, the request is already satisfied. Not when the
7025            // configuration changed since that spawn: then the newer process
7026            // predates the spec this restart may exist to apply.
7027            // Generation publication can lag a child that is already serving
7028            // while its identity is recorded. Only process creation after request
7029            // receipt satisfies a restart, not publication after receipt.
7030            let satisfied_by_generation = if initiated
7031                && child
7032                    .as_ref()
7033                    .is_some_and(|child| child.spawned_at > queued_at)
7034            {
7035                lock_snapshot(snapshot).ok().and_then(|state| {
7036                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7037                })
7038            } else {
7039                None
7040            };
7041            let satisfied_by_pending = initiated
7042                && child.is_none()
7043                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7044                    // No child exists yet, so this replacement will necessarily
7045                    // be created after the queued request, unlike a published
7046                    // generation whose process may already have been serving.
7047                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7048                    if pending {
7049                        state.coalesced_restart_pending = true;
7050                    }
7051                    pending
7052                });
7053            if satisfied_by_pending {
7054                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7055            } else if let Some(generation) = satisfied_by_generation {
7056                info!(
7057                    module_id = %spec.module_id,
7058                    received_at_generation,
7059                    "restart already satisfied by generation {generation}; not restarting again"
7060                );
7061            } else if initiated {
7062                // Precedence: this restart's operator override, else the module's
7063                // configured budget (already resolved into the runtime).
7064                let drain_timeout = drain_timeout_ms
7065                    .map(Duration::from_millis)
7066                    .unwrap_or(runtime.drain_timeout);
7067                if let Err(err) = restart_child(
7068                    spec,
7069                    runtime,
7070                    registry,
7071                    process_liveness,
7072                    snapshot,
7073                    child,
7074                    drain_timeout,
7075                )
7076                .await
7077                {
7078                    warn!(
7079                        module_id = %spec.module_id,
7080                        error = %err,
7081                        "operator restart failed after initiation ack; module state carries the outcome"
7082                    );
7083                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7084                        state.state = ModuleState::Failed;
7085                        clear_current_process_facts(state);
7086                    });
7087                } else {
7088                    runtime.start_requester = requester;
7089                    runtime.pending_activity_restart = true;
7090                }
7091            }
7092            true
7093        }
7094        SupervisorCommand::Reload { requester, reply } => {
7095            let result =
7096                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7097            if result.is_ok() {
7098                runtime.start_requester = requester;
7099                runtime.pending_activity_restart = true;
7100            }
7101            if result.is_ok()
7102                && runtime
7103                    .scheduled_respawn
7104                    .lock()
7105                    .unwrap_or_else(|p| p.into_inner())
7106                    .as_ref()
7107                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7108            {
7109                *runtime
7110                    .deferred_reload_reply
7111                    .lock()
7112                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7113            } else {
7114                let _ = reply.send(result);
7115            }
7116            true
7117        }
7118        SupervisorCommand::SetEnabled {
7119            enabled,
7120            requester,
7121            reply,
7122        } => {
7123            let previous_requester = runtime.start_requester.clone();
7124            if enabled {
7125                runtime.start_requester = requester.clone();
7126            }
7127            let result = set_child_enabled(
7128                spec,
7129                runtime,
7130                registry,
7131                process_liveness,
7132                snapshot,
7133                child,
7134                enabled,
7135            )
7136            .await;
7137            if matches!(result, Ok(true)) {
7138                runtime.pending_activity_restart = false;
7139                if enabled {
7140                    runtime.start_requester = requester;
7141                } else {
7142                    runtime.start_requester = None;
7143                    runtime.record_activity_transition(
7144                        "subc.module_disabled",
7145                        &spec.module_id,
7146                        requester,
7147                    );
7148                }
7149            } else {
7150                runtime.start_requester = previous_requester;
7151            }
7152            let _ = reply.send(result);
7153            true
7154        }
7155        SupervisorCommand::UpdateConfiguration {
7156            spec: next_spec,
7157            health,
7158            drain_timeout_ms,
7159            reply,
7160        } => {
7161            if let Some(handle) = &runtime.supervisor_handle {
7162                handle.apply_identity_configuration(&next_spec);
7163            }
7164            *spec = next_spec;
7165            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7166                state.configuration_updated_since_spawn = true;
7167            });
7168            let health_changed = runtime.health != health;
7169            runtime.health = health;
7170            // Reset the cadence and old endpoint's failure streak on a live
7171            // health-policy change rather than waiting for its old deadline.
7172            if health_changed {
7173                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7174                    state.health = ModuleHealthStatus::default();
7175                });
7176            }
7177            runtime.drain_timeout = drain_timeout_ms
7178                .map(Duration::from_millis)
7179                .unwrap_or(runtime.default_drain_timeout);
7180            *runtime
7181                .effective_drain_timeout
7182                .lock()
7183                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7184            let _ = reply.send(());
7185            true
7186        }
7187        SupervisorCommand::Swap {
7188            requester,
7189            ready_timeout,
7190            reply,
7191        } => {
7192            let mut swap_runtime = runtime.clone();
7193            swap_runtime.start_requester = requester;
7194            let before = lock_snapshot(snapshot)
7195                .ok()
7196                .map(|state| state.spawn_generation);
7197            let end = swap::run_swap(
7198                spec,
7199                &swap_runtime,
7200                registry,
7201                process_liveness,
7202                snapshot,
7203                child,
7204                commands,
7205                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7206                reply,
7207            )
7208            .await;
7209            if before
7210                != lock_snapshot(snapshot)
7211                    .ok()
7212                    .map(|state| state.spawn_generation)
7213            {
7214                runtime.start_requester = swap_runtime.start_requester;
7215            }
7216            requeued.extend(end.requeue);
7217            true
7218        }
7219    }
7220}
7221
7222async fn restart_child(
7223    spec: &ModuleSpec,
7224    runtime: &SupervisorRuntimeConfig,
7225    registry: &Registry,
7226    process_liveness: &SupervisorProcessLiveness,
7227    snapshot: &SharedSnapshot,
7228    child: &mut Option<SupervisedChild>,
7229    drain_timeout: Duration,
7230) -> Result<(), SuperviseError> {
7231    // Restart cycles a running module; it must not silently start a disabled one.
7232    if !lock_snapshot(snapshot)?.enabled {
7233        return Err(SuperviseError::Disabled {
7234            module_id: spec.module_id.clone(),
7235        });
7236    }
7237    let stop_notice = begin_forwarding_drain_with_timeout(
7238        spec,
7239        runtime,
7240        registry,
7241        snapshot,
7242        None,
7243        RouteCloseReason::Restart,
7244        drain_timeout,
7245    )
7246    .await?;
7247
7248    if child.is_some() {
7249        drain_optional_child(
7250            &spec.module_id,
7251            spec.protocol,
7252            stop_notice,
7253            registry,
7254            runtime.forwarding.as_deref(),
7255            snapshot,
7256            &runtime.terminal_ring,
7257            &runtime.spawn_events,
7258            child,
7259            drain_timeout,
7260            ModuleState::Restarting,
7261            Some(true),
7262        )
7263        .await?;
7264    } else {
7265        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7266            state.enabled = true;
7267            state.state = ModuleState::Restarting;
7268            clear_current_process_facts(state);
7269        })?;
7270        release_dead_registration(
7271            registry,
7272            runtime.forwarding.as_deref(),
7273            snapshot,
7274            &spec.module_id,
7275        )
7276        .await?;
7277    }
7278
7279    reset_restart_count(snapshot, &spec.module_id)?;
7280    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7281    schedule_respawn(
7282        runtime,
7283        snapshot,
7284        &spec.module_id,
7285        runtime.restart_policy.backoff,
7286        RespawnKind::Spawn,
7287    )
7288}
7289
7290async fn reload_child(
7291    spec: &ModuleSpec,
7292    runtime: &SupervisorRuntimeConfig,
7293    registry: &Registry,
7294    process_liveness: &SupervisorProcessLiveness,
7295    snapshot: &SharedSnapshot,
7296    child: &mut Option<SupervisedChild>,
7297) -> Result<(), SuperviseError> {
7298    // Reload cycles a running module; it must not silently start a disabled one.
7299    if !lock_snapshot(snapshot)?.enabled {
7300        return Err(SuperviseError::Disabled {
7301            module_id: spec.module_id.clone(),
7302        });
7303    }
7304    let stop_notice = begin_forwarding_drain(
7305        spec,
7306        runtime,
7307        registry,
7308        snapshot,
7309        Some(true),
7310        RouteCloseReason::Reload,
7311    )
7312    .await?;
7313
7314    if child.is_some() {
7315        drain_optional_child(
7316            &spec.module_id,
7317            spec.protocol,
7318            stop_notice,
7319            registry,
7320            runtime.forwarding.as_deref(),
7321            snapshot,
7322            &runtime.terminal_ring,
7323            &runtime.spawn_events,
7324            child,
7325            runtime.drain_timeout,
7326            ModuleState::Restarting,
7327            Some(true),
7328        )
7329        .await?;
7330    } else {
7331        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7332            state.enabled = true;
7333            state.state = ModuleState::Restarting;
7334            clear_current_process_facts(state);
7335        })?;
7336        release_dead_registration(
7337            registry,
7338            runtime.forwarding.as_deref(),
7339            snapshot,
7340            &spec.module_id,
7341        )
7342        .await?;
7343    }
7344
7345    reset_restart_count(snapshot, &spec.module_id)?;
7346    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7347    schedule_respawn(
7348        runtime,
7349        snapshot,
7350        &spec.module_id,
7351        runtime.restart_policy.backoff,
7352        RespawnKind::Reload,
7353    )
7354}
7355
7356async fn finish_reload_child(
7357    spec: &ModuleSpec,
7358    runtime: &SupervisorRuntimeConfig,
7359    registry: &Registry,
7360    process_liveness: &SupervisorProcessLiveness,
7361    snapshot: &SharedSnapshot,
7362    child: &mut Option<SupervisedChild>,
7363) -> Result<(), SuperviseError> {
7364    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7365    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7366        Ok(next_child) => next_child,
7367        Err(err) => {
7368            return handle_reload_spawn_failure(
7369                spec,
7370                runtime,
7371                process_liveness,
7372                snapshot,
7373                child,
7374                format!("new child failed to spawn: {err}"),
7375            )
7376            .await;
7377        }
7378    };
7379    *child = Some(next_child);
7380
7381    let wait_outcome = {
7382        let active_child = child.as_mut().expect("new reload child was just stored");
7383        wait_for_registration_after_reload(
7384            registry,
7385            &spec.module_id,
7386            snapshot,
7387            active_child,
7388            REGISTRY_RELEASE_TIMEOUT,
7389        )
7390        .await?
7391    };
7392
7393    match wait_outcome {
7394        RegistrationWaitOutcome::Registered => {
7395            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7396            Ok(())
7397        }
7398        RegistrationWaitOutcome::Exited(exit_report) => {
7399            if let Some(active_child) = child.as_mut() {
7400                active_child.drain_stderr(&spec.module_id).await;
7401            }
7402            // Keep the reaped child's roster guard until its terminal is written.
7403            // Shutdown waits on that guard, not on the child Option used for respawn.
7404            let mut exited_child = child.take().expect("exited reload child is still stored");
7405            #[cfg(test)]
7406            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7407                gate.reached.notify_one();
7408                gate.resume.notified().await;
7409            }
7410            let result = handle_reload_child_registration_failure(
7411                spec,
7412                runtime,
7413                registry,
7414                process_liveness,
7415                snapshot,
7416                child,
7417                ReloadRegistrationFailure {
7418                    exit_report: registration_failure_exit_report(exit_report),
7419                    reason: exited_child
7420                        .spawn_failure
7421                        .clone()
7422                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7423                },
7424            )
7425            .await;
7426            exited_child.release_roster();
7427            result
7428        }
7429        RegistrationWaitOutcome::TimedOut => {
7430            let mut timed_out_child = child
7431                .take()
7432                .expect("timed-out reload child is still running");
7433            timed_out_child
7434                .start_kill()
7435                .map_err(|source| SuperviseError::Kill {
7436                    module_id: spec.module_id.clone(),
7437                    source,
7438                })?;
7439            let status = timed_out_child
7440                .wait()
7441                .await
7442                .map_err(|source| SuperviseError::Wait {
7443                    module_id: spec.module_id.clone(),
7444                    source,
7445                })?;
7446            timed_out_child.drain_stderr(&spec.module_id).await;
7447            handle_reload_child_registration_failure(
7448                spec,
7449                runtime,
7450                registry,
7451                process_liveness,
7452                snapshot,
7453                child,
7454                ReloadRegistrationFailure {
7455                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7456                        snapshot,
7457                        &timed_out_child,
7458                        &status,
7459                    )),
7460                    reason: format!(
7461                        "new child did not register within {:?}",
7462                        REGISTRY_RELEASE_TIMEOUT
7463                    ),
7464                },
7465            )
7466            .await
7467        }
7468    }
7469}
7470
7471async fn set_child_enabled(
7472    spec: &ModuleSpec,
7473    runtime: &SupervisorRuntimeConfig,
7474    registry: &Registry,
7475    process_liveness: &SupervisorProcessLiveness,
7476    snapshot: &SharedSnapshot,
7477    child: &mut Option<SupervisedChild>,
7478    enabled: bool,
7479) -> Result<bool, SuperviseError> {
7480    let (current_enabled, current_state, respawn_pending) = {
7481        let state = lock_snapshot(snapshot)?;
7482        (state.enabled, state.state, state.respawn_pending)
7483    };
7484    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7485    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7486    // clean (Stopped) has no live process and no other in-band recovery — the
7487    // operator's start is the explicit recovery act and resets the budget. Without
7488    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7489    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7490    // the one providing every agent's shell.
7491    let revive_terminal = enabled
7492        && current_enabled
7493        && child.is_none()
7494        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7495            || (current_state == ModuleState::Restarting && !respawn_pending));
7496    if current_enabled == enabled && !revive_terminal {
7497        return Ok(false);
7498    }
7499
7500    if enabled {
7501        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7502            state.enabled = true;
7503            state.state = ModuleState::Starting;
7504            clear_current_process_facts(state);
7505        })?;
7506        #[cfg(test)]
7507        if runtime.test_seed_stale_facts_before_enable_spawn {
7508            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7509                state.process_alive = true;
7510                state.pid = Some(41);
7511                state.spawned_at_ms = Some(42);
7512                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7513                state.spawned_file_identity = Some(SpawnedFileIdentity {
7514                    device: 43,
7515                    inode: 44,
7516                    #[cfg(windows)]
7517                    windows_image: None,
7518                });
7519            })?;
7520        }
7521        release_dead_registration(
7522            registry,
7523            runtime.forwarding.as_deref(),
7524            snapshot,
7525            &spec.module_id,
7526        )
7527        .await?;
7528        reset_restart_count(snapshot, &spec.module_id)?;
7529        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7530        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7531            Ok(next_child) => next_child,
7532            Err(err) => {
7533                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7534                    state.state = ModuleState::Failed;
7535                    clear_current_process_facts(state);
7536                }) {
7537                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7538                }
7539                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7540                return Err(err);
7541            }
7542        };
7543        *child = Some(next_child);
7544        debug!(module_id = %spec.module_id, "supervised module enabled");
7545        Ok(true)
7546    } else {
7547        let stop_notice = begin_forwarding_drain_if_configured(
7548            spec,
7549            runtime,
7550            registry,
7551            snapshot,
7552            Some(false),
7553            RouteCloseReason::Disable,
7554        )
7555        .await?;
7556        drain_optional_child(
7557            &spec.module_id,
7558            spec.protocol,
7559            stop_notice,
7560            registry,
7561            runtime.forwarding.as_deref(),
7562            snapshot,
7563            &runtime.terminal_ring,
7564            &runtime.spawn_events,
7565            child,
7566            runtime.drain_timeout,
7567            ModuleState::Disabled,
7568            Some(false),
7569        )
7570        .await?;
7571        debug!(module_id = %spec.module_id, "supervised module disabled");
7572        Ok(true)
7573    }
7574}
7575
7576#[allow(clippy::too_many_arguments)]
7577async fn on_child_exit(
7578    spec: &ModuleSpec,
7579    policy: RestartPolicy,
7580    registry: &Registry,
7581    snapshot: &SharedSnapshot,
7582    terminal_ring: &Arc<Mutex<TerminalRing>>,
7583    spawn_events: &SpawnEventFeed,
7584    roster: &ChildRoster,
7585    exit_report: ExitReport,
7586) -> NextAction {
7587    // Once the daemon has begun shutting down, no exit is a crash to recover
7588    // from: the module is exiting because the daemon is going away (EOF on its
7589    // connection, or a service manager signalling the whole cgroup). Record it
7590    // as such and never schedule a respawn, which would only start a process
7591    // for the shutdown to end again.
7592    if roster.is_closed() {
7593        return on_child_exit_during_daemon_shutdown(
7594            spec,
7595            registry,
7596            snapshot,
7597            terminal_ring,
7598            spawn_events,
7599            exit_report,
7600        )
7601        .await;
7602    }
7603    // Every stop the supervisor itself asks for (operator stop, disable,
7604    // restart, reload, swap, a health restart, a drain that runs out of budget)
7605    // takes the child out of the supervise loop and reaps it in
7606    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7607    // that reaches this point was not requested by the daemon.
7608    //
7609    // For a subc-wire module a clean exit is still a stop: those modules are
7610    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7611    // crash, and exiting 0 is a deliberate choice the module made. A
7612    // `protocol: "none"` module is a stock program we cannot change, and many
7613    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7614    // stop would leave the module down for good after any stray signal, so it
7615    // goes through the crash path instead: it spends restart budget, respawns
7616    // with the crash backoff, and ends `failed` when the budget runs out.
7617    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7618        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7619    match exit_report.kind {
7620        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7621            info!(
7622                module_id = %spec.module_id,
7623                exit_code = ?exit_report.code,
7624                exit_signal = ?exit_report.signal,
7625                "supervised module exited cleanly"
7626            );
7627            if let Err(err) = publish_after_terminal_record(
7628                &spec.module_id,
7629                || {
7630                    record_terminal(
7631                        &spec.module_id,
7632                        terminal_ring,
7633                        spawn_events,
7634                        &exit_report,
7635                        TerminalDisposition::Stopped,
7636                    )
7637                },
7638                || {
7639                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7640                        state.state = ModuleState::Stopped;
7641                        clear_current_process_facts(state);
7642                        state.last_exit = Some(exit_report.clone());
7643                    })
7644                },
7645            ) {
7646                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7647            }
7648            let registration_released = match wait_for_registration_release(
7649                registry,
7650                &spec.module_id,
7651                REGISTRY_RELEASE_TIMEOUT,
7652            )
7653            .await
7654            {
7655                Ok(()) => true,
7656                Err(err) => {
7657                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7658                    false
7659                }
7660            };
7661            NextAction::Stop {
7662                registration_released,
7663            }
7664        }
7665        ExitKind::Clean | ExitKind::Crash => {
7666            if unrequested_clean_exit_of_protocol_none {
7667                warn!(
7668                    module_id = %spec.module_id,
7669                    exit_code = ?exit_report.code,
7670                    exit_signal = ?exit_report.signal,
7671                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7672                );
7673            } else {
7674                warn!(
7675                    module_id = %spec.module_id,
7676                    exit_code = ?exit_report.code,
7677                    exit_signal = ?exit_report.signal,
7678                    "supervised module exited abnormally (crash)"
7679                );
7680            }
7681            let mut restart_schedule = None;
7682            let mut disposition = TerminalDisposition::Disabled;
7683            let mut final_state = ModuleState::Disabled;
7684            // Set only when the budget is what stopped the module, so the
7685            // terminal record says which limit was hit rather than leaving
7686            // `failed` to be read as "crashed once, badly".
7687            let mut disposition_detail = lock_snapshot(snapshot)
7688                .ok()
7689                .and_then(|mut state| state.spawn_failure.take());
7690            let now = Instant::now();
7691            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7692                if state.enabled {
7693                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7694                        // Reserving the last slot exhausts the budget for the
7695                        // replacement, not for this exit. Publish its scheduled
7696                        // recovery atomically with the reservation so connection
7697                        // cleanup sees it while terminal recording still holds
7698                        // the visible state at Running.
7699                        state.respawn_pending = true;
7700                        final_state = ModuleState::Restarting;
7701                        restart_schedule = Some(schedule);
7702                        disposition = TerminalDisposition::Restarting;
7703                    } else {
7704                        final_state = ModuleState::Failed;
7705                        disposition = TerminalDisposition::Failed;
7706                        let budget = policy.budget_exhausted_detail();
7707                        disposition_detail =
7708                            Some(disposition_detail.take().map_or_else(
7709                                || budget.clone(),
7710                                |cause| format!("{cause}; {budget}"),
7711                            ));
7712                    }
7713                } else {
7714                    disposition = TerminalDisposition::Disabled;
7715                }
7716            }) {
7717                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7718                return NextAction::Stop {
7719                    registration_released: false,
7720                };
7721            }
7722            if disposition == TerminalDisposition::Failed {
7723                // The window is in the message, not only in the fields: this line
7724                // is read in a scrollback where a bare `max_restarts=3` reads as a
7725                // lifetime cap and sends the operator looking for three crashes
7726                // that never happened together.
7727                error!(
7728                    module_id = %spec.module_id,
7729                    max_restarts = policy.max_restarts,
7730                    window_secs = policy.window.as_secs(),
7731                    "module stopped: {}",
7732                    policy.budget_exhausted_detail()
7733                );
7734            }
7735            if let Err(err) = publish_after_terminal_record(
7736                &spec.module_id,
7737                || {
7738                    record_terminal_with_detail(
7739                        &spec.module_id,
7740                        terminal_ring,
7741                        spawn_events,
7742                        &exit_report,
7743                        disposition,
7744                        disposition_detail,
7745                    )
7746                },
7747                || {
7748                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7749                        state.state = final_state;
7750                        clear_current_process_facts(state);
7751                        state.last_exit = Some(exit_report.clone());
7752                    })
7753                },
7754            ) {
7755                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7756            }
7757
7758            if let Some(schedule) = restart_schedule {
7759                NextAction::Restart {
7760                    schedule: Some(schedule),
7761                }
7762            } else {
7763                let registration_released = match wait_for_registration_release(
7764                    registry,
7765                    &spec.module_id,
7766                    REGISTRY_RELEASE_TIMEOUT,
7767                )
7768                .await
7769                {
7770                    Ok(()) => true,
7771                    Err(err) => {
7772                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7773                        false
7774                    }
7775                };
7776                NextAction::Stop {
7777                    registration_released,
7778                }
7779            }
7780        }
7781        ExitKind::DeliberateSeverance => {
7782            warn!(
7783                module_id = %spec.module_id,
7784                exit_code = ?exit_report.code,
7785                exit_signal = ?exit_report.signal,
7786                "supervised module exited after deliberate connection severance"
7787            );
7788            let should_restart = match lock_snapshot(snapshot) {
7789                Ok(state) => state.enabled,
7790                Err(err) => {
7791                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7792                    return NextAction::Stop {
7793                        registration_released: false,
7794                    };
7795                }
7796            };
7797            let final_state = if should_restart {
7798                ModuleState::Restarting
7799            } else {
7800                ModuleState::Disabled
7801            };
7802            if let Err(err) = publish_after_terminal_record(
7803                &spec.module_id,
7804                || {
7805                    record_terminal(
7806                        &spec.module_id,
7807                        terminal_ring,
7808                        spawn_events,
7809                        &exit_report,
7810                        terminal_disposition(final_state),
7811                    )
7812                },
7813                || {
7814                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7815                        state.state = final_state;
7816                        clear_current_process_facts(state);
7817                        state.last_exit = Some(exit_report.clone());
7818                        state.lifetime_restarts += 1;
7819                    })
7820                },
7821            ) {
7822                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7823                return NextAction::Stop {
7824                    registration_released: false,
7825                };
7826            }
7827
7828            if should_restart {
7829                NextAction::Restart { schedule: None }
7830            } else {
7831                let registration_released = match wait_for_registration_release(
7832                    registry,
7833                    &spec.module_id,
7834                    REGISTRY_RELEASE_TIMEOUT,
7835                )
7836                .await
7837                {
7838                    Ok(()) => true,
7839                    Err(err) => {
7840                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7841                        false
7842                    }
7843                };
7844                NextAction::Stop {
7845                    registration_released,
7846                }
7847            }
7848        }
7849    }
7850}
7851
7852async fn on_child_exit_during_daemon_shutdown(
7853    spec: &ModuleSpec,
7854    registry: &Registry,
7855    snapshot: &SharedSnapshot,
7856    terminal_ring: &Arc<Mutex<TerminalRing>>,
7857    spawn_events: &SpawnEventFeed,
7858    exit_report: ExitReport,
7859) -> NextAction {
7860    info!(
7861        module_id = %spec.module_id,
7862        exit_code = ?exit_report.code,
7863        exit_signal = ?exit_report.signal,
7864        exit_kind = ?exit_report.kind,
7865        "supervised module exited during daemon shutdown; not restarting it"
7866    );
7867    if let Err(err) = publish_after_terminal_record(
7868        &spec.module_id,
7869        || {
7870            record_terminal(
7871                &spec.module_id,
7872                terminal_ring,
7873                spawn_events,
7874                &exit_report,
7875                TerminalDisposition::DaemonShutdown,
7876            )
7877        },
7878        || {
7879            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7880                state.state = ModuleState::Stopped;
7881                clear_current_process_facts(state);
7882                state.last_exit = Some(exit_report.clone());
7883            })
7884        },
7885    ) {
7886        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7887    }
7888    let registration_released =
7889        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7890            .await
7891            .is_ok();
7892    NextAction::Stop {
7893        registration_released,
7894    }
7895}
7896
7897// Keep the status lock out of terminal I/O. Readers may observe the previous
7898// state while recording is in progress, but never a completed exit without its
7899// history. Journal errors are logged by the journal and retained in the ring;
7900// a recording panic still publishes the final state before resuming the unwind.
7901fn publish_after_terminal_record<T>(
7902    module_id: &str,
7903    record: impl FnOnce(),
7904    publish: impl FnOnce() -> T,
7905) -> T {
7906    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7907    let published = publish();
7908    if let Err(panic) = recorded {
7909        error!(
7910            module_id,
7911            "terminal recording panicked; published final state before resuming unwind"
7912        );
7913        std::panic::resume_unwind(panic);
7914    }
7915    published
7916}
7917
7918fn fail_after_wait_error(
7919    module_id: &str,
7920    snapshot: &SharedSnapshot,
7921    terminal_ring: &Arc<Mutex<TerminalRing>>,
7922    spawn_events: &SpawnEventFeed,
7923    detail: String,
7924) {
7925    let report = wait_error_exit_report();
7926    publish_after_terminal_record(
7927        module_id,
7928        || {
7929            record_terminal_with_detail(
7930                module_id,
7931                terminal_ring,
7932                spawn_events,
7933                &report,
7934                TerminalDisposition::Failed,
7935                Some(detail),
7936            )
7937        },
7938        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7939    );
7940}
7941
7942#[cfg(test)]
7943fn record_wait_error_terminal(
7944    module_id: &str,
7945    terminal_ring: &Arc<Mutex<TerminalRing>>,
7946    spawn_events: &SpawnEventFeed,
7947) {
7948    record_terminal(
7949        module_id,
7950        terminal_ring,
7951        spawn_events,
7952        &wait_error_exit_report(),
7953        TerminalDisposition::Failed,
7954    );
7955}
7956
7957fn record_terminal(
7958    module_id: &str,
7959    terminal_ring: &Arc<Mutex<TerminalRing>>,
7960    spawn_events: &SpawnEventFeed,
7961    exit_report: &ExitReport,
7962    disposition: TerminalDisposition,
7963) {
7964    record_terminal_with_detail(
7965        module_id,
7966        terminal_ring,
7967        spawn_events,
7968        exit_report,
7969        disposition,
7970        None,
7971    );
7972}
7973
7974/// The ring lock is held only to capture the read (see
7975/// `TerminalJournal::capture_read`), so this module's exits keep recording
7976/// while the journal files are read. Blocking: it reads files.
7977fn durable_terminal_history_of(
7978    terminal_ring: &Mutex<TerminalRing>,
7979    module_id: &str,
7980) -> subc_control::TerminalHistory {
7981    let read = terminal_ring
7982        .lock()
7983        .unwrap_or_else(|p| p.into_inner())
7984        .capture_durable_history();
7985    read.read(module_id)
7986}
7987
7988fn record_terminal_with_detail(
7989    module_id: &str,
7990    terminal_ring: &Arc<Mutex<TerminalRing>>,
7991    spawn_events: &SpawnEventFeed,
7992    exit_report: &ExitReport,
7993    disposition: TerminalDisposition,
7994    disposition_detail: Option<String>,
7995) {
7996    #[cfg(test)]
7997    terminal_history_tests::before_terminal_record(spawn_events);
7998    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7999    let record = TerminalRecord {
8000        exit_code: exit_report.code,
8001        exit_signal: exit_report.signal,
8002        at_ms: exit_report.at_ms,
8003        disposition,
8004        exit_kind: exit_report.kind.into(),
8005        disposition_detail,
8006    };
8007    terminal_ring
8008        .lock()
8009        .unwrap_or_else(|poisoned| poisoned.into_inner())
8010        .record_exit(module_id, record);
8011}
8012
8013fn untrack_if_registration_released(
8014    process_liveness: &SupervisorProcessLiveness,
8015    registry: &Registry,
8016    module_id: &str,
8017    snapshot: &SharedSnapshot,
8018) {
8019    match registry.get_module(module_id) {
8020        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8021        Ok(Some(_)) => {}
8022        Err(err) => {
8023            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8024        }
8025    }
8026}
8027
8028/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8029/// then apply the module's configured entries minus daemon-private capture keys.
8030///
8031/// Separated from `spawn_child` only so it can be asserted without spawning a
8032/// process — a duplicate of this logic in a test would pass while the real one
8033/// drifted, which is the defect class this function exists to avoid.
8034/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8035/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8036/// either and the argument would stop a stock binary from starting at all.
8037/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8038///
8039/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8040/// through [`apply_wire_spawn_args_for_role`].
8041#[cfg(test)]
8042fn apply_wire_spawn_args(
8043    command: &mut Command,
8044    spec: &ModuleSpec,
8045    connection_file_path: Option<&std::path::Path>,
8046    handle: Option<&SupervisorHandle>,
8047) -> Result<Option<NonceHandoff>, SuperviseError> {
8048    apply_wire_spawn_args_for_role(
8049        command,
8050        spec,
8051        connection_file_path,
8052        handle,
8053        SpawnRole::Plain,
8054    )
8055}
8056
8057/// The read end of a spawn's launch-nonce pipe, prepared by
8058/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8059/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8060/// server before spawn and authenticates its client after the child exists.
8061#[cfg(unix)]
8062type NonceHandoff = subc_os::LaunchNonceHandoff;
8063#[cfg(windows)]
8064type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8065#[cfg(not(any(unix, windows)))]
8066type NonceHandoff = std::convert::Infallible;
8067
8068/// Create the launch nonce that a new module process must present when it
8069/// registers or opens a consumer connection, including a replacement process
8070/// started alongside a still-serving module.
8071///
8072/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8073/// a separate candidate token so the still-serving incumbent and its consumers
8074/// keep their nonce. Both records are installed before the process exists, so
8075/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8076///
8077/// On Unix the nonce is delivered only through a pipe. It is written into
8078/// a pipe whose read end the child gets as descriptor 3, named by
8079/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8080/// process of the same user cannot read it with `ps eww`. That handoff is
8081/// returned rather than installed here, because installing it replaces
8082/// whatever the child has at descriptor 3 and so must be the last pre-exec
8083/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8084/// the Linux callback that writes the process into its cgroup. Only after that
8085/// does `handoff.install_last` register the descriptor-3 replacement, so it
8086/// cannot close a descriptor the cgroup callback still needs.
8087/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8088fn apply_wire_spawn_args_for_role(
8089    command: &mut Command,
8090    spec: &ModuleSpec,
8091    connection_file_path: Option<&std::path::Path>,
8092    handle: Option<&SupervisorHandle>,
8093    role: SpawnRole,
8094) -> Result<Option<NonceHandoff>, SuperviseError> {
8095    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8096    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8097    // included: a daemon started from a module's process tree inherits it,
8098    // and passing it on would point the child at a descriptor it does not
8099    // have.
8100    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8101    #[cfg(windows)]
8102    {
8103        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8104        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8105    }
8106    // Remove inherited or configured copies too: withholding must mean absent.
8107    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8108    if spec.protocol == ModuleProtocol::None {
8109        return Ok(None);
8110    }
8111    if let Some(connection_file_path) = connection_file_path {
8112        command.arg(SUBC_ARG).arg(connection_file_path);
8113    }
8114
8115    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8116    // route.open attestation. Reserved modules additionally use the same nonce
8117    // for HELLO id-squatting protection. A respawn rotates both records.
8118    let nonce = generate_launch_nonce()?;
8119    if let Some(handle) = handle {
8120        match role {
8121            SpawnRole::Plain => {
8122                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8123                if spec.reserved {
8124                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8125                }
8126            }
8127            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8128        }
8129    }
8130    #[cfg(unix)]
8131    let handoff = {
8132        let handoff =
8133            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8134                program: spec.program.clone(),
8135                source,
8136                cgroup_path: None,
8137            })?;
8138        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8139        Some(handoff)
8140    };
8141    #[cfg(windows)]
8142    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8143    #[cfg(not(any(unix, windows)))]
8144    let handoff = None;
8145    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8146    // until the daemon's per-spawn delivery records show that every running
8147    // Windows module has read its secret from the named pipe.
8148    #[cfg(not(unix))]
8149    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8150    Ok(handoff)
8151}
8152
8153fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8154    command.env_remove(CK_LOG_ENV);
8155    // The spawn role is the supervisor's to set, and only on a swap candidate
8156    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8157    // it, is what makes it absent on a plain spawn: the daemon's own
8158    // environment could carry it, and so could a spec built outside daemon
8159    // config (config refuses it as an `env` key). A module reading it on a
8160    // plain restart would pick the long swap budget and leave callers waiting.
8161    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8162    for (key, value) in &spec.env {
8163        // cortexkit-log currently exposes retention only as a Rust struct, not
8164        // environment names. These values are daemon-private sink metadata and
8165        // must never become a public child-process contract by being inherited.
8166        if matches!(
8167            key.as_str(),
8168            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8169        ) || key == SUBC_SPAWN_ROLE_ENV
8170        {
8171            continue;
8172        }
8173        command.env(key, value);
8174    }
8175}
8176
8177#[cfg(windows)]
8178fn prepare_windows_nonce_pipe(
8179    command: &mut Command,
8180    spec: &ModuleSpec,
8181    nonce: &str,
8182) -> Option<NonceHandoff> {
8183    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8184        Ok(handoff) => {
8185            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8186            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8187            // still register until every module reads the pipe. A reader without
8188            // this variable set to env returns the pipe error instead of using
8189            // an inherited or configured environment copy.
8190            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8191            Some(handoff)
8192        }
8193        Err(error) => {
8194            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8195            None
8196        }
8197    }
8198}
8199
8200#[cfg(windows)]
8201fn start_windows_nonce_pipe(
8202    handoff: Option<NonceHandoff>,
8203    pid: u32,
8204    spec: &ModuleSpec,
8205) -> Option<subc_os::LaunchNoncePipeDelivery> {
8206    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8207    // same maximum wait used for registration during a reload. Older modules
8208    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8209    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8210        Ok(delivery) => Some(delivery),
8211        Err(error) => {
8212            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8213            None
8214        }
8215    })
8216}
8217
8218fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8219    #[cfg(windows)]
8220    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8221        return Some(
8222            if snapshot
8223                .pipe_consumed
8224                .as_ref()
8225                .is_some_and(|consumed| consumed.get().is_some())
8226            {
8227                "pipe"
8228            } else {
8229                "env"
8230            }
8231            .to_owned(),
8232        );
8233    }
8234    let _ = snapshot;
8235    None
8236}
8237
8238/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8239/// of a blue/green swap.
8240#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8241enum SpawnRole {
8242    Plain,
8243    SwapCandidate,
8244}
8245
8246/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8247/// `apply_child_env` has already removed the variable for every spawn.
8248fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8249    if role == SpawnRole::SwapCandidate {
8250        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8251    }
8252}
8253
8254fn previous_instance_error(
8255    spec: &ModuleSpec,
8256    pid: u32,
8257    reason: crate::live_children::RetainedReason,
8258) -> SuperviseError {
8259    SuperviseError::Spawn {
8260        program: spec.program.clone(),
8261        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8262        cgroup_path: None,
8263    }
8264}
8265
8266fn spawn_child(
8267    spec: &ModuleSpec,
8268    connection_file_path: Option<&std::path::Path>,
8269    handle: Option<&SupervisorHandle>,
8270    ring: &Arc<Mutex<StderrRing>>,
8271    capture_logs_dir: Option<&std::path::Path>,
8272    roster: &ChildRoster,
8273    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8274) -> Result<SupervisedChild, SuperviseError> {
8275    spawn_child_in_slot(
8276        spec,
8277        connection_file_path,
8278        handle,
8279        ring,
8280        capture_logs_dir,
8281        roster,
8282        #[cfg(target_os = "linux")]
8283        cgroup_placement,
8284        SpawnRole::Plain,
8285        false,
8286    )
8287}
8288
8289/// Spawn one process of `spec` into a slot.
8290///
8291/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8292/// A swap candidate needs a different cgroup from the process it is replacing,
8293/// which is still alive: in the same cgroup the two would be one kill domain,
8294/// and killing a failed candidate could take the incumbent with it.
8295///
8296/// The stderr capture file is `<module_id>.stderr.log` for every process of
8297/// the module, whichever slot it is in, because that is the one file
8298/// `ck module logs` reads. During a swap's overlap both processes append to it;
8299/// the daemon writes whole lines, so the two interleave by line, which is also
8300/// the merged view an operator wants while a swap runs.
8301#[allow(clippy::too_many_arguments)]
8302fn spawn_child_in_slot(
8303    spec: &ModuleSpec,
8304    connection_file_path: Option<&std::path::Path>,
8305    handle: Option<&SupervisorHandle>,
8306    ring: &Arc<Mutex<StderrRing>>,
8307    capture_logs_dir: Option<&std::path::Path>,
8308    roster: &ChildRoster,
8309    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8310    role: SpawnRole,
8311    alternate_slot: bool,
8312) -> Result<SupervisedChild, SuperviseError> {
8313    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8314        return Err(previous_instance_error(spec, pid, reason));
8315    }
8316    if roster.is_closed() {
8317        return Err(SuperviseError::Spawn {
8318            program: spec.program.clone(),
8319            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8320            cgroup_path: None,
8321        });
8322    }
8323    #[cfg(target_os = "linux")]
8324    let cgroup_name = {
8325        // Slot names alone are not kill domains: a retired incumbent may still
8326        // be draining when a later enable/restart spawns into the same slot.
8327        // Decimal entropy keeps the suffix unambiguous; Placement performs
8328        // the module-id escaping and constructs the filesystem path.
8329        if cgroup_placement.is_none() {
8330            swap::cgroup_name(&spec.module_id, alternate_slot)
8331        } else {
8332            let nonce = generate_launch_nonce()?;
8333            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8334            // Leave room for byte escaping and the suffix under NAME_MAX. The
8335            // label is only for humans; the nonce identifies the kill domain.
8336            let mut end = spec.module_id.len().min(64);
8337            while !spec.module_id.is_char_boundary(end) {
8338                end -= 1;
8339            }
8340            format!(
8341                "{}_{suffix}",
8342                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8343            )
8344        }
8345    };
8346    #[cfg(not(target_os = "linux"))]
8347    let _ = alternate_slot;
8348    #[cfg(target_os = "macos")]
8349    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8350    #[cfg(windows)]
8351    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8352        program: spec.program.clone(),
8353        source,
8354        cgroup_path: None,
8355    })?;
8356    #[cfg(windows)]
8357    let mut command = Command::new(image_capture.path());
8358    #[cfg(not(any(target_os = "macos", windows)))]
8359    let mut command = Command::new(&spec.program);
8360    command.args(&spec.args);
8361    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8362    // that is the whole of the intent, so remove that one key rather than the
8363    // environment.
8364    //
8365    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8366    // and took the POSIX environment with it. Modules spawned that way had no
8367    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8368    // logging:
8369    //
8370    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8371    //     both unset it fell back to the temp dir alone and `ck` could not find
8372    //     a daemon running on the same machine from inside any module's process
8373    //     tree — reporting a path the file has never lived at, which reads as
8374    //     "the daemon did not write its file".
8375    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8376    //     the RELATIVE `.local/share`, so a module deriving its own store path
8377    //     resolved it against its own CWD. That is the store-fragmentation
8378    //     defect the daemon already refuses in config (`parse_doc` rejects a
8379    //     relative `storage.data_home`) arriving by derivation instead.
8380    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8381    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8382    //     quietly rather than erroring.
8383    //
8384    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8385    // offered one candidate under /tmp while the file sat in /run/user/1000.
8386    //
8387    // A configured module is unaffected either way: `module_spec()` puts the
8388    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8389    // wins over anything ambient.
8390    apply_child_env(&mut command, spec);
8391    apply_spawn_role(&mut command, role);
8392    let nonce_handoff =
8393        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8394
8395    #[cfg(target_os = "linux")]
8396    let cgroup_path = cgroup_placement
8397        .map(|placement| placement.module_path(&cgroup_name))
8398        .transpose()
8399        .map_err(|source| SuperviseError::Cgroup {
8400            module_id: spec.module_id.clone(),
8401            source,
8402        })?;
8403    #[cfg(not(target_os = "linux"))]
8404    let cgroup_path: Option<PathBuf> = None;
8405    #[cfg(target_os = "linux")]
8406    if let Some(path) = &cgroup_path {
8407        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8408            if let Some(placement) = cgroup_placement {
8409                remove_module_cgroup(placement, &cgroup_name);
8410            }
8411            return Err(error);
8412        }
8413    }
8414
8415    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8416        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8417        match ChildOutputSink::open(&path, capture_retention(spec)) {
8418            Ok(sink) => sink,
8419            Err(error) => {
8420                warn!(
8421                    module_id = %spec.module_id,
8422                    path = %path.display(),
8423                    error = %error,
8424                    "could not open child output capture file; forwarding to stderr"
8425                );
8426                ChildOutputSink::Stderr
8427            }
8428        }
8429    } else {
8430        ChildOutputSink::Stderr
8431    };
8432
8433    command.stdout(Stdio::piped());
8434    command.stderr(Stdio::piped());
8435    command.kill_on_drop(true);
8436    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8437    // before exec). In the daemon's group, a service manager that kills the
8438    // job's process group when the daemon exits (launchd's default) killed
8439    // every module at the same moment its control connection closed, so no
8440    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8441    // module is reached only by the daemon: the EOF it sees when its
8442    // connection closes, and the bounded stop in `child_roster` for anything
8443    // still running after that. On Linux this composes with the cgroup
8444    // placement above: that is a pre_exec write to cgroup.procs, std performs
8445    // setpgid in the child before running pre_exec callbacks, and the two
8446    // change independent process attributes.
8447    //
8448    // stdin is /dev/null because a process outside the terminal's foreground
8449    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8450    // by hand would otherwise hand down. Under a service manager stdin is
8451    // already /dev/null.
8452    #[cfg(unix)]
8453    command.process_group(0);
8454    command.stdin(Stdio::null());
8455    // The LAST pre-exec step, after the cgroup placement above: installing the
8456    // nonce at descriptor 3 replaces whatever the child had there, which could
8457    // be the descriptor an earlier step writes through.
8458    #[cfg(unix)]
8459    if let Some(handoff) = nonce_handoff {
8460        handoff.install_last(command.as_std_mut());
8461    }
8462    #[cfg(not(any(unix, windows)))]
8463    let _ = nonce_handoff;
8464
8465    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8466    // cannot run a single instruction -- and therefore cannot spawn a
8467    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8468    // other two steps and why the window matters.
8469    #[cfg(windows)]
8470    {
8471        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8472            program: spec.program.clone(),
8473            source,
8474            cgroup_path: None,
8475        })?;
8476        subc_jobobject::suspend_on_create_async(&mut command);
8477    }
8478    // Timestamp before the spawn call: on Unix the child can serve before that
8479    // call even returns. This conservative bound never labels an older child
8480    // as a replacement for a request received during process creation.
8481    let spawned_at = Instant::now();
8482    let mut child = match command.spawn() {
8483        Ok(child) => child,
8484        Err(source) => {
8485            #[cfg(target_os = "linux")]
8486            if let Some(placement) = cgroup_placement {
8487                remove_module_cgroup(placement, &cgroup_name);
8488            }
8489            return Err(SuperviseError::Spawn {
8490                program: spec.program.clone(),
8491                source,
8492                cgroup_path,
8493            });
8494        }
8495    };
8496    #[cfg(test)]
8497    spawn_record_tests::after_create(&child, spec);
8498    // The parent must close its writer now: the acknowledgement pipe reports EOF
8499    // only when every writer is gone, and the child's copy closes when the
8500    // trampoline replaces itself with the module. Command holds only an integer
8501    // in its pre_exec callback, not another writer.
8502    #[cfg(target_os = "macos")]
8503    drop(exec_ack);
8504
8505    // Bind while the child is still suspended, before job assignment resumes it.
8506    // Keeping the executable handle open across CreateProcessW denies rename
8507    // and deletion, so the file cannot be exchanged during spawn. Bind that
8508    // file ID to the process handle returned by spawn, never a reopened PID.
8509    #[cfg(windows)]
8510    let windows_image = match image_capture.bind_tokio(&child) {
8511        Ok(image) => Arc::new(image),
8512        Err(source) => {
8513            kill_and_reap_rejected_child(&mut child);
8514            return Err(SuperviseError::Spawn {
8515                program: spec.program.clone(),
8516                source,
8517                cgroup_path: None,
8518            });
8519        }
8520    };
8521    // Assign the suspended child to its module job; resume only after its
8522    // process identity has been durably recorded for orphan recovery.
8523    #[cfg(windows)]
8524    let job = contain_spawned_child(&mut child, spec)?;
8525    let spawned_at_ms = unix_ms_now();
8526    #[cfg(not(windows))]
8527    let spawned_from = spec.program.clone();
8528    #[cfg(windows)]
8529    let spawned_from = windows_image.path().to_path_buf();
8530    #[cfg(not(windows))]
8531    let spawned_file_identity = spawned_file_identity(&spawned_from);
8532    #[cfg(windows)]
8533    let spawned_file_identity = Some(SpawnedFileIdentity {
8534        device: 0,
8535        inode: 0,
8536        windows_image: Some(Arc::clone(&windows_image)),
8537    });
8538    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8539        program: spec.program.clone(),
8540        source: io::Error::other("spawned child exposed no live pid"),
8541        cgroup_path: cgroup_path.clone(),
8542    })?;
8543    #[cfg(windows)]
8544    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8545    #[cfg(not(windows))]
8546    let process_start_time = crate::provenance::process_start_time(pid);
8547    #[cfg(windows)]
8548    let process_start_time = Some(windows_image.start_time());
8549    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8550    #[cfg(all(test, target_os = "macos"))]
8551    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8552    // Record the committed kernel image, not the configured pathname: PATH
8553    // lookup and shebang interpretation can select a different executable.
8554    // Linux's posix_spawn can return before that image is installed, so the
8555    // observer also waits for the kernel's first-exec flag to clear.
8556    #[cfg(not(windows))]
8557    let recorded_image = observe_spawned_image(pid);
8558    // spawn() confirms only the first exec, into the trampoline. Never persist
8559    // the trampoline image; the asynchronous acknowledgement publishes the
8560    // module image once the trampoline has replaced itself with the module.
8561    #[cfg(target_os = "macos")]
8562    let recorded_image = if privacy_exec.is_some() {
8563        None
8564    } else {
8565        recorded_image
8566    };
8567    #[cfg(target_os = "linux")]
8568    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8569    #[cfg(not(target_os = "linux"))]
8570    let recorded_cgroup_name = None;
8571    let roster_guard = roster.admit(
8572        spec.module_id.clone(),
8573        pid,
8574        spec.protocol,
8575        process_start_time,
8576        crate::child_roster::RecordedIdentity {
8577            #[cfg(not(windows))]
8578            start_time: recorded_image.map(|image| image.start_time),
8579            #[cfg(windows)]
8580            start_time: process_start_time,
8581            #[cfg(not(windows))]
8582            executable: recorded_image
8583                .and_then(|image| image.executable)
8584                .map(crate::live_children::ExecutableIdentity::from),
8585            #[cfg(windows)]
8586            executable: Some(crate::live_children::ExecutableIdentity::from(
8587                windows_image.identity(),
8588            )),
8589            cgroup_name: recorded_cgroup_name,
8590            #[cfg(target_os = "linux")]
8591            cgroup_placement: cgroup_placement.cloned(),
8592        },
8593    );
8594    let roster_guard = match roster_guard {
8595        Ok(guard) => guard,
8596        Err(source) => {
8597            #[cfg(windows)]
8598            if let Some(job) = &job {
8599                let _ = job.terminate();
8600            }
8601            #[cfg(target_os = "linux")]
8602            kill_module_cgroup(cgroup_placement, &cgroup_name);
8603            kill_and_reap_rejected_child(&mut child);
8604            #[cfg(target_os = "linux")]
8605            if let Some(placement) = cgroup_placement {
8606                remove_module_cgroup(placement, &cgroup_name);
8607            }
8608            return Err(SuperviseError::Spawn {
8609                program: spec.program.clone(),
8610                source: io::Error::other(format!(
8611                    "could not durably record module child: {source}"
8612                )),
8613                cgroup_path,
8614            });
8615        }
8616    };
8617    // The check at the top of this function can pass just before daemon
8618    // shutdown begins, and the process is only in the roster from here on.
8619    // The shutdown stop returns as soon as it finds the roster empty, so a
8620    // process admitted after that look would outlive the daemon. The roster
8621    // is closed before the stop first reads it and admission happens under
8622    // the roster's lock, so either the stop sees this process or this check
8623    // sees the roster closed: end the process now rather than start a module
8624    // the daemon is about to stop.
8625    if roster.is_closed() {
8626        // This child was never admitted, so there is no module protocol shutdown to wait for.
8627        #[cfg(target_os = "linux")]
8628        kill_module_cgroup(cgroup_placement, &cgroup_name);
8629        if let Err(error) = child.start_kill() {
8630            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8631        }
8632        #[cfg(target_os = "linux")]
8633        if let Some(placement) = cgroup_placement {
8634            // This spawn was never admitted, so shutdown has no roster entry
8635            // to await. Do not detach its cleanup: the runtime could exit
8636            // before that task reaps the rejected child and removes its group.
8637            while matches!(child.try_wait(), Ok(None)) {
8638                std::thread::yield_now();
8639            }
8640            if matches!(
8641                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8642                subc_cgroup::KillOutcome::Killed
8643            ) {
8644                if let Ok(path) = placement.module_path(&cgroup_name) {
8645                    while std::fs::read_to_string(path.join("cgroup.events"))
8646                        .ok()
8647                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8648                    {
8649                        std::thread::yield_now();
8650                    }
8651                }
8652            }
8653            remove_module_cgroup(placement, &cgroup_name);
8654        }
8655        drop(roster_guard);
8656        return Err(SuperviseError::Spawn {
8657            program: spec.program.clone(),
8658            source: io::Error::other(
8659                "the daemon began shutting down while this process was starting; ended it",
8660            ),
8661            cgroup_path,
8662        });
8663    }
8664
8665    #[cfg(windows)]
8666    if let Err(error) = resume_suspended_child(pid, spec) {
8667        kill_and_reap_rejected_child(&mut child);
8668        return Err(error);
8669    }
8670
8671    let stdout_pump = match child.stdout.take() {
8672        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8673        None => {
8674            warn!(
8675                module_id = %spec.module_id,
8676                "spawned child exposed no stdout pipe; file capture will be incomplete"
8677            );
8678            None
8679        }
8680    };
8681    let stderr_pump = match child.stderr.take() {
8682        Some(stderr) => {
8683            let generation = ring
8684                .lock()
8685                .unwrap_or_else(|poisoned| poisoned.into_inner())
8686                .begin_process();
8687            Some(StderrPump {
8688                task: tokio::spawn(pump_stderr_to(
8689                    stderr,
8690                    Arc::clone(ring),
8691                    generation,
8692                    output_sink,
8693                )),
8694                generation,
8695            })
8696        }
8697        None => {
8698            // Spawning succeeded but the pipe did not materialise. Recording it as
8699            // uncaptured keeps the tail honest: the alternative is an empty tail
8700            // that reads as a module which printed nothing.
8701            ring.lock()
8702                .unwrap_or_else(|poisoned| poisoned.into_inner())
8703                .mark_not_captured("stderr pipe was not available on spawn");
8704            warn!(
8705                module_id = %spec.module_id,
8706                "spawned child exposed no stderr pipe; tail will be unavailable"
8707            );
8708            None
8709        }
8710    };
8711
8712    let exec = Arc::new(crate::activity::ExecOutcome::new());
8713    #[cfg(target_os = "macos")]
8714    let pending_exec = privacy_exec.is_some();
8715    #[cfg(not(target_os = "macos"))]
8716    let pending_exec = false;
8717    if !pending_exec {
8718        exec.finish(crate::activity::ExecState::CommittedExec);
8719    }
8720    Ok(SupervisedChild {
8721        child,
8722        exec: crate::activity::ExecOwner(exec),
8723        #[cfg(windows)]
8724        nonce_delivery,
8725        protocol: spec.protocol,
8726        #[cfg(target_os = "linux")]
8727        module_id: cgroup_name,
8728        #[cfg(target_os = "linux")]
8729        cgroup_placement: cgroup_placement.cloned(),
8730        #[cfg(windows)]
8731        job,
8732        stdout_pump,
8733        stderr_pump,
8734        stderr_ring: Arc::clone(ring),
8735        spawned_at,
8736        spawned_at_ms,
8737        spawned_from,
8738        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8739        spawned_file_identity,
8740        process_start_time,
8741        process_identity,
8742        pid,
8743        roster_guard: Some(roster_guard),
8744        #[cfg(target_os = "macos")]
8745        privacy_exec,
8746        #[cfg(target_os = "macos")]
8747        report_ready: Arc::new(OnceLock::new()),
8748        spawn_failure: None,
8749    })
8750}
8751
8752#[cfg(test)]
8753mod spawn_record_tests {
8754    use super::*;
8755
8756    thread_local! {
8757        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8758    }
8759
8760    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8761        LAST_PID.set(child.id());
8762        #[cfg(windows)]
8763        if let Some((_, address)) = _spec
8764            .env
8765            .iter()
8766            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8767        {
8768            use std::io::{Read, Write};
8769            let mut stream = std::net::TcpStream::connect(address).unwrap();
8770            stream
8771                .write_all(&child.id().unwrap().to_ne_bytes())
8772                .unwrap();
8773            let mut release = [0];
8774            stream.read_exact(&mut release).unwrap();
8775        }
8776    }
8777
8778    #[cfg(windows)]
8779    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8780        if spec
8781            .env
8782            .iter()
8783            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8784        {
8785            Err(io::Error::other("injected module job assignment failure"))
8786        } else {
8787            Ok(())
8788        }
8789    }
8790
8791    fn spec() -> ModuleSpec {
8792        ModuleSpec {
8793            module_id: "spawn-record-worker".into(),
8794            #[cfg(unix)]
8795            program: "/bin/sleep".into(),
8796            #[cfg(windows)]
8797            program: std::env::current_exe().unwrap(),
8798            #[cfg(unix)]
8799            args: vec!["60".into()],
8800            #[cfg(windows)]
8801            args: vec![
8802                "--exact".into(),
8803                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8804                "--nocapture".into(),
8805            ],
8806            env: vec![],
8807            reserved: false,
8808            reserved_prefixes: vec![],
8809            protocol: ModuleProtocol::None,
8810            overlap: Default::default(),
8811        }
8812    }
8813
8814    #[tokio::test]
8815    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8816        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8817        let supervisor =
8818            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8819                .with_live_children_record(home.join("live-children.json"));
8820        crate::live_children::fail_next_file_sync();
8821        let result = supervisor.spawn(spec());
8822        let failed = result.is_err();
8823        if let Ok(module) = result {
8824            module.stop().await.unwrap();
8825        }
8826        assert!(failed, "record failure must fail spawn");
8827        let pid = LAST_PID.get().expect("the real OS child was created");
8828        assert!(
8829            subc_os::Process::open(pid).unwrap().is_none(),
8830            "child {pid} survived rejected admission or was not reaped"
8831        );
8832    }
8833
8834    #[cfg(target_os = "macos")]
8835    #[test]
8836    #[ignore = "manual on-device latency measurement"]
8837    fn durable_record_write_latency_50() {
8838        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8839        let path = home.join("live-children.json");
8840        let children = vec![crate::live_children::LiveChild {
8841            module_id: "latency-worker".into(),
8842            pid: std::process::id(),
8843            protocol: ModuleProtocol::None,
8844            start_time: Some(1),
8845            executable: None,
8846            cgroup_name: None,
8847        }];
8848        let mut samples = Vec::new();
8849        for _ in 0..50 {
8850            let start = std::time::Instant::now();
8851            crate::live_children::write_record(&path, &children).unwrap();
8852            samples.push(start.elapsed());
8853        }
8854        samples.sort();
8855        eprintln!(
8856            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8857            (samples[24] + samples[25]) / 2,
8858            samples[47]
8859        );
8860    }
8861
8862    #[cfg(windows)]
8863    #[test]
8864    fn windows_store_touch_fixture() {
8865        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8866            std::fs::write(path, b"module ran").unwrap();
8867        }
8868    }
8869
8870    #[cfg(windows)]
8871    #[tokio::test]
8872    async fn windows_daemon_fixture() {
8873        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8874            return;
8875        };
8876        let mut spec = spec();
8877        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8878        let supervisor =
8879            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8880        let _module = supervisor.spawn(spec).unwrap();
8881    }
8882
8883    #[cfg(windows)]
8884    #[test]
8885    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8886        use std::io::Read;
8887        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8888        // An outer job models a scheduler/service host. The daemon's root and
8889        // root job must nest without requiring breakaway from this job.
8890        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8891        command
8892            .args([
8893                "--exact",
8894                "supervise::spawn_record_tests::windows_daemon_fixture",
8895                "--nocapture",
8896            ])
8897            .env(
8898                "SUBC_TEST_DAEMON_BARRIER",
8899                listener.local_addr().unwrap().to_string(),
8900            );
8901        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8902        listener.set_nonblocking(true).unwrap();
8903        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8904        let mut stream = loop {
8905            if let Ok((stream, _)) = listener.accept() {
8906                break stream;
8907            }
8908            assert!(
8909                daemon.child.try_wait().unwrap().is_none(),
8910                "daemon failed before rendezvous"
8911            );
8912            assert!(
8913                std::time::Instant::now() < deadline,
8914                "daemon did not reach CreateProcess failpoint"
8915            );
8916            std::thread::yield_now();
8917        };
8918        stream
8919            .set_read_timeout(Some(Duration::from_secs(30)))
8920            .unwrap();
8921        let mut pid = [0; 4];
8922        stream.read_exact(&mut pid).unwrap();
8923        let pid = u32::from_ne_bytes(pid);
8924        assert!(subc_jobobject::process_exists(pid));
8925        // Terminate only the daemon, not its outer job: otherwise the host job
8926        // would also kill a child whose daemon never joined its root job.
8927        daemon.child.kill().unwrap();
8928        daemon.child.wait().unwrap();
8929        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8930        if !exited {
8931            daemon.job.terminate().unwrap();
8932        }
8933        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8934    }
8935
8936    #[cfg(windows)]
8937    #[tokio::test]
8938    async fn windows_containment_failure_never_resumes() {
8939        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8940        let sentinel = home.join("store-touched");
8941        let mut spec = spec();
8942        spec.env
8943            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8944        spec.env.push((
8945            "SUBC_TEST_STORE_TOUCH".into(),
8946            sentinel.display().to_string(),
8947        ));
8948        let supervisor =
8949            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8950        let result = supervisor.spawn(spec);
8951        // A version that resumes despite assignment failure must have time to
8952        // execute its sentinel before dropping the returned module kills it.
8953        if let Ok(module) = result {
8954            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8955                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8956                    tokio::task::yield_now().await;
8957                }
8958            })
8959            .await;
8960            let _ = module.stop().await;
8961            panic!(
8962                "assignment failure resumed a module; store touched={}",
8963                sentinel.exists()
8964            );
8965        }
8966        assert!(
8967            !sentinel.exists(),
8968            "refused module executed store-touch sentinel"
8969        );
8970        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8971    }
8972
8973    #[cfg(windows)]
8974    #[tokio::test]
8975    async fn windows_nested_daemon_fixture() {
8976        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8977            return;
8978        };
8979        let mut spec = spec();
8980        spec.env.push((
8981            "SUBC_TEST_STORE_TOUCH".into(),
8982            sentinel.to_str().unwrap().into(),
8983        ));
8984        let supervisor =
8985            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8986        let module = supervisor
8987            .spawn(spec)
8988            .expect("per-module job must nest below daemon and host jobs");
8989        tokio::time::timeout(Duration::from_secs(10), async {
8990            while !PathBuf::from(&sentinel).exists() {
8991                tokio::task::yield_now().await;
8992            }
8993        })
8994        .await
8995        .unwrap();
8996        module.stop().await.unwrap();
8997    }
8998
8999    #[cfg(windows)]
9000    #[test]
9001    fn windows_module_job_nests_under_host_job() {
9002        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9003        let sentinel = home.join("nested-store-touched");
9004        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9005        command
9006            .args([
9007                "--exact",
9008                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9009                "--nocapture",
9010            ])
9011            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9012        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9013        assert!(daemon.child.wait().unwrap().success());
9014        assert!(sentinel.exists(), "nested protocol:none module never ran");
9015    }
9016}
9017
9018#[cfg(target_os = "linux")]
9019pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9020    use subc_cgroup::KillOutcome;
9021    match subc_cgroup::kill_module(placement, module_id) {
9022        KillOutcome::Killed => {}
9023        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9024            debug!(
9025                module_id,
9026                "cgroup tree kill unavailable; using direct-child kill"
9027            );
9028        }
9029        KillOutcome::IoError { path, error } => {
9030            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9031        }
9032    }
9033}
9034
9035/// Assign a freshly spawned Windows child without ever resuming on failure.
9036///
9037/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9038/// child assigned **while it is still suspended** (step 1 is
9039/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9040/// the child's roster admission is durable. Failed containment kills and reaps
9041/// the suspended child instead of running an unprotected module.
9042#[cfg(windows)]
9043fn contain_spawned_child(
9044    child: &mut Child,
9045    spec: &ModuleSpec,
9046) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9047    let assignment = || {
9048        let job = subc_jobobject::JobObject::new()?;
9049        #[cfg(test)]
9050        spawn_record_tests::assignment_failpoint(spec)?;
9051        job.assign(child)?;
9052        Ok::<_, io::Error>(job)
9053    };
9054    match assignment() {
9055        Ok(job) => Ok(Some(job)),
9056        Err(source) => {
9057            kill_and_reap_rejected_child(child);
9058            Err(SuperviseError::Spawn {
9059                program: spec.program.clone(),
9060                source: io::Error::other(format!(
9061                    "Windows module job containment failed: {source}"
9062                )),
9063                cgroup_path: None,
9064            })
9065        }
9066    }
9067}
9068
9069/// Rejection is synchronous: do not return a spawn error while the child can
9070/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9071fn kill_and_reap_rejected_child(child: &mut Child) {
9072    #[cfg(unix)]
9073    if let Some(pid) = child
9074        .id()
9075        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9076    {
9077        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9078    }
9079    let _ = child.start_kill();
9080    while matches!(child.try_wait(), Ok(None)) {
9081        std::thread::yield_now();
9082    }
9083}
9084
9085/// Resume a suspended child, killing it if it cannot be started.
9086///
9087/// A suspended process holds a pid and does nothing, so there is no useful
9088/// state to return: the caller gets an error and the spawn fails.
9089#[cfg(windows)]
9090fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9091    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9092        // Kill it here rather than leaving a suspended process for the caller
9093        // to notice; `kill_on_drop` would eventually do this, but the module
9094        // would have been reported as running in between.
9095        let _ = std::process::Command::new("taskkill.exe")
9096            .args(["/PID", &pid.to_string(), "/T", "/F"])
9097            .stdin(Stdio::null())
9098            .stdout(Stdio::null())
9099            .stderr(Stdio::null())
9100            .status();
9101        return Err(SuperviseError::Spawn {
9102            program: spec.program.clone(),
9103            source,
9104            cgroup_path: None,
9105        });
9106    }
9107    Ok(())
9108}
9109
9110#[cfg(target_os = "linux")]
9111fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9112    match placement.remove_module(module_id) {
9113        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9114        Err(error) => warn!(
9115            module_id,
9116            error = %error,
9117            "could not remove module cgroup after process exit; continuing teardown"
9118        ),
9119    }
9120}
9121
9122#[cfg(target_os = "linux")]
9123async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9124    // Reaping the direct child is not proof its descendants exited. End the
9125    // residual tree and wait for the kernel's population fact before rmdir;
9126    // otherwise a successful parent wait leaks a directory on each restart.
9127    if matches!(
9128        subc_cgroup::kill_module(Some(placement), module_id),
9129        subc_cgroup::KillOutcome::Killed
9130    ) {
9131        if let Ok(path) = placement.module_path(module_id) {
9132            while std::fs::read_to_string(path.join("cgroup.events"))
9133                .ok()
9134                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9135            {
9136                sleep(Duration::from_millis(1)).await;
9137            }
9138        }
9139    }
9140    remove_module_cgroup(placement, module_id);
9141}
9142
9143#[cfg(target_os = "linux")]
9144fn apply_cgroup_placement(
9145    command: &mut Command,
9146    spec: &ModuleSpec,
9147    path: &std::path::Path,
9148) -> Result<(), SuperviseError> {
9149    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9150        module_id: spec.module_id.clone(),
9151        source,
9152    })
9153}
9154
9155fn capture_retention(spec: &ModuleSpec) -> Retention {
9156    let defaults = Retention::default();
9157    let value = |name: &str| {
9158        spec.env
9159            .iter()
9160            .rev()
9161            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9162    };
9163    Retention {
9164        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9165            .and_then(|value| value.parse().ok())
9166            .unwrap_or(defaults.max_file_mb),
9167        keep: value(CAPTURE_KEEP_ENV)
9168            .and_then(|value| value.parse().ok())
9169            .unwrap_or(defaults.keep),
9170        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9171            .and_then(|value| value.parse().ok())
9172            .unwrap_or(defaults.max_age_days),
9173    }
9174}
9175
9176/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9177/// module's registration to the exact process the supervisor spawned.
9178fn generate_launch_nonce() -> Result<String, SuperviseError> {
9179    let mut bytes = [0u8; 32];
9180    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9181        reason: source.to_string(),
9182    })?;
9183    let mut hex = String::with_capacity(64);
9184    for b in bytes {
9185        use std::fmt::Write;
9186        let _ = write!(hex, "{b:02x}");
9187    }
9188    Ok(hex)
9189}
9190
9191/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9192/// signal about how many leading bytes matched.
9193fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9194    if a.len() != b.len() {
9195        return false;
9196    }
9197    let mut diff = 0u8;
9198    for (x, y) in a.iter().zip(b.iter()) {
9199        diff |= x ^ y;
9200    }
9201    diff == 0
9202}
9203
9204/// The kernel's image after an acknowledged exec, shared by ordinary launches
9205/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9206/// not identities inferred from a configured pathname.
9207#[cfg(not(windows))]
9208fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9209    let process = subc_os::Process::open(pid).ok().flatten()?;
9210    #[cfg(target_os = "linux")]
9211    return sample_after_linux_exec(
9212        || {
9213            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9214            let committed = linux_exec_committed(&stat);
9215            #[cfg(test)]
9216            spawn_image_tests::report_exec_state(committed);
9217            committed
9218        },
9219        || process.observe(),
9220    );
9221    #[cfg(not(target_os = "linux"))]
9222    process.observe()
9223}
9224
9225/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9226/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9227/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9228#[cfg(target_os = "linux")]
9229fn linux_exec_committed(stat: &str) -> Option<bool> {
9230    const PF_FORKNOEXEC: u64 = 0x40;
9231    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9232    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9233}
9234
9235#[cfg(target_os = "linux")]
9236fn sample_after_linux_exec(
9237    mut exec_committed: impl FnMut() -> Option<bool>,
9238    observe: impl FnOnce() -> Option<subc_os::Observation>,
9239) -> Option<subc_os::Observation> {
9240    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9241    while !exec_committed()? {
9242        // A failed or indefinitely stalled exec must not publish the inherited
9243        // image as authority to signal a future orphan. Missing identity keeps
9244        // the existing conservative orphan-sweep behavior.
9245        if std::time::Instant::now() >= deadline {
9246            return None;
9247        }
9248        std::thread::sleep(Duration::from_micros(50));
9249    }
9250    observe()
9251}
9252
9253#[cfg(all(test, target_os = "linux"))]
9254mod spawn_image_tests {
9255    use super::*;
9256    use std::cell::{Cell, RefCell};
9257
9258    enum ExecEvent {
9259        Uncommitted,
9260        Image(Option<subc_os::Observation>),
9261    }
9262
9263    thread_local! {
9264        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9265    }
9266
9267    pub(super) fn report_exec_state(committed: Option<bool>) {
9268        if committed == Some(false) {
9269            EXEC_EVENTS.with(|events| {
9270                if let Some(events) = events.borrow_mut().take() {
9271                    let _ = events.send(ExecEvent::Uncommitted);
9272                }
9273            });
9274        }
9275    }
9276
9277    #[test]
9278    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9279        use std::io::{Read, Write};
9280        use std::os::fd::AsRawFd;
9281
9282        let descriptor = std::fs::File::open("/dev/null").unwrap();
9283        let mut command = std::process::Command::new("/bin/sleep");
9284        command.arg("30");
9285        let (mut ready, mut release) =
9286            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9287                .unwrap();
9288        let spawn = std::thread::spawn(move || command.spawn());
9289        let mut message = [0u8; 8];
9290        ready.read_exact(&mut message).unwrap();
9291        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9292        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9293        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9294        let (events, received) = std::sync::mpsc::channel();
9295        let observer = std::thread::spawn(move || {
9296            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9297            let image = observe_spawned_image(pid);
9298            let _ = events.send(ExecEvent::Image(image));
9299        });
9300        // Release only once the production observer has seen the uncommitted
9301        // exec, or has incorrectly returned the inherited image. Either way,
9302        // release and reap before asserting so a regression cannot leak a child.
9303        let first = received.recv_timeout(Duration::from_secs(5));
9304        release.write_all(&[1]).unwrap();
9305        let mut child = spawn.join().unwrap().unwrap();
9306        let after = match first {
9307            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9308                Ok(ExecEvent::Image(image)) => image,
9309                _ => None,
9310            },
9311            Ok(ExecEvent::Image(image)) => image,
9312            Err(_) => None,
9313        };
9314        observer.join().unwrap();
9315        child.kill().unwrap();
9316        child.wait().unwrap();
9317
9318        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9319        assert_eq!(
9320            inherited.unwrap().executable,
9321            subc_os::file_identity(&std::env::current_exe().unwrap())
9322        );
9323        assert_eq!(
9324            after.unwrap().executable,
9325            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9326        );
9327    }
9328
9329    #[test]
9330    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9331        let committed = Cell::new(false);
9332        let inherited = subc_os::Observation {
9333            start_time: 23,
9334            executable: Some(subc_os::FileIdentity {
9335                device: 29,
9336                inode: 31,
9337            }),
9338        };
9339        let module = subc_os::Observation {
9340            start_time: 23,
9341            executable: Some(subc_os::FileIdentity {
9342                device: 29,
9343                inode: 37,
9344            }),
9345        };
9346        let polls = Cell::new(0);
9347        let image = sample_after_linux_exec(
9348            || {
9349                polls.set(polls.get() + 1);
9350                // Hold the inherited image through the first poll, as in the
9351                // posix_spawn parent-wake window, then commit the module image.
9352                committed.set(polls.get() == 2);
9353                Some(committed.get())
9354            },
9355            || Some(if committed.get() { module } else { inherited }),
9356        );
9357        assert_eq!(image, Some(module));
9358        assert_eq!(polls.get(), 2);
9359    }
9360
9361    #[test]
9362    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9363        assert_eq!(
9364            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9365            Some(false)
9366        );
9367        assert_eq!(
9368            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9369            Some(true)
9370        );
9371        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9372        assert_eq!(linux_exec_committed("17 (a) R"), None);
9373    }
9374
9375    #[test]
9376    fn unreadable_exec_state_never_samples_an_inherited_image() {
9377        assert_eq!(
9378            sample_after_linux_exec(
9379                || None,
9380                || panic!("must not sample before exec confirmation")
9381            ),
9382            None
9383        );
9384    }
9385
9386    #[test]
9387    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9388        assert_eq!(
9389            sample_after_linux_exec(
9390                || Some(false),
9391                || panic!("must not sample after an exec timeout")
9392            ),
9393            None
9394        );
9395    }
9396}
9397
9398/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9399/// different executable after the no-delete capture was taken.
9400#[cfg(windows)]
9401fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9402    let mut program = spec.program.clone();
9403    if program.is_absolute() || program.components().count() > 1 {
9404        // An explicit path can name a PE image with no extension. Try that
9405        // exact file first, as Command does, before falling back to .exe.
9406        match subc_os::ExecutableCapture::open(&program) {
9407            Ok(capture) => return Ok(capture),
9408            Err(error)
9409                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9410            Err(error) => return Err(error),
9411        }
9412        program.set_extension("exe");
9413        return subc_os::ExecutableCapture::open(&program);
9414    }
9415    if program.extension().is_none() {
9416        program.set_extension("exe");
9417    }
9418    let custom_path = spec
9419        .env
9420        .iter()
9421        .rev()
9422        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9423    let mut directories = Vec::new();
9424    // These are Windows' default executable search locations. A configured
9425    // PATH still overrides the inherited PATH, not the system directories.
9426    if let Some(parent) = std::env::current_exe()?.parent() {
9427        directories.push(parent.to_path_buf());
9428    }
9429    directories.push(std::env::current_dir()?);
9430    if let Some(windows) = std::env::var_os("SystemRoot") {
9431        let windows = PathBuf::from(windows);
9432        directories.push(windows.join("System32"));
9433        directories.push(windows.clone());
9434    }
9435    let path = custom_path
9436        .map(|(_, value)| std::ffi::OsString::from(value))
9437        .or_else(|| std::env::var_os("PATH"));
9438    if let Some(path) = path {
9439        directories.extend(std::env::split_paths(&path));
9440    }
9441    for directory in directories {
9442        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9443            Ok(capture) => return Ok(capture),
9444            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9445            Err(error) => return Err(error),
9446        }
9447    }
9448    Err(io::Error::new(
9449        io::ErrorKind::NotFound,
9450        "resolved executable not found",
9451    ))
9452}
9453
9454fn spawn_and_mark_running(
9455    spec: &ModuleSpec,
9456    runtime: &SupervisorRuntimeConfig,
9457    snapshot: &SharedSnapshot,
9458) -> Result<SupervisedChild, SuperviseError> {
9459    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9460    let child = spawn_child(
9461        spec,
9462        runtime.connection_file_path.as_deref(),
9463        runtime.supervisor_handle.as_ref(),
9464        &runtime.stderr_ring,
9465        runtime.capture_logs_dir.as_deref(),
9466        &runtime.child_roster,
9467        #[cfg(target_os = "linux")]
9468        runtime.cgroup_placement.as_ref(),
9469    )?;
9470    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9471    if let Some(spawn) = activity_spawn {
9472        spawn.complete(snapshot);
9473    }
9474    Ok(child)
9475}
9476
9477enum RegistrationWaitOutcome {
9478    Registered,
9479    Exited(ExitReport),
9480    TimedOut,
9481}
9482
9483struct ReloadRegistrationFailure {
9484    exit_report: ExitReport,
9485    reason: String,
9486}
9487
9488#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9489enum BusyGaugeObservation {
9490    Quiescent,
9491    Busy,
9492    Omitted,
9493}
9494
9495/// What one health answer said about one declared busy gauge.
9496#[derive(Debug, Clone, PartialEq, Eq)]
9497enum GaugeVerdict {
9498    /// The gauge read 0: this gauge is not what holds the drain.
9499    Zero,
9500    /// The gauge read a positive count.
9501    NonZero(u64),
9502    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9503    /// a negative number). It counts as busy, because the daemon cannot tell
9504    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9505    NotACount(String),
9506    /// The health answer had no metrics object, or one without this gauge.
9507    Missing,
9508    /// The probe produced no health answer to read; carries the probe error's
9509    /// evidence label (`no-answer` for a probe that timed out).
9510    ProbeFailed(&'static str),
9511}
9512
9513impl GaugeVerdict {
9514    fn reads_quiet(&self) -> bool {
9515        matches!(self, Self::Zero)
9516    }
9517}
9518
9519impl fmt::Display for GaugeVerdict {
9520    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9521        match self {
9522            Self::Zero => f.write_str("zero"),
9523            Self::NonZero(value) => write!(f, "non_zero({value})"),
9524            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9525            Self::Missing => f.write_str("missing_from_health_body"),
9526            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9527        }
9528    }
9529}
9530
9531/// The longest JSON text kept for a gauge value that is not a count, so a
9532/// module reporting a large object as a gauge cannot flood the drain log.
9533const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9534
9535/// Read each declared gauge out of one health answer's metrics.
9536fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9537    let metrics = metrics.and_then(Value::as_object);
9538    gauges
9539        .iter()
9540        .map(
9541            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9542                None => GaugeVerdict::Missing,
9543                Some(value) => match value.as_u64() {
9544                    Some(0) => GaugeVerdict::Zero,
9545                    Some(count) => GaugeVerdict::NonZero(count),
9546                    None => {
9547                        let mut text = value.to_string();
9548                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9549                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9550                            while !text.is_char_boundary(end) {
9551                                end -= 1;
9552                            }
9553                            text.truncate(end);
9554                            text.push_str("...");
9555                        }
9556                        GaugeVerdict::NotACount(text)
9557                    }
9558                },
9559            },
9560        )
9561        .collect()
9562}
9563
9564/// The drain's overall reading of one health answer: omitted when any
9565/// declared gauge is missing, quiescent only when every gauge reads 0.
9566fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9567    if verdicts
9568        .iter()
9569        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9570    {
9571        BusyGaugeObservation::Omitted
9572    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9573        BusyGaugeObservation::Quiescent
9574    } else {
9575        BusyGaugeObservation::Busy
9576    }
9577}
9578
9579/// One declared gauge's history over a drain: the latest verdict on it, and
9580/// the latest value actually read for it (which a later failed probe or
9581/// missing gauge does not erase). Times are offsets from the start of the
9582/// drain's wait.
9583#[derive(Debug, Clone, Default, PartialEq, Eq)]
9584struct GaugeTrace {
9585    latest: Option<(GaugeVerdict, Duration)>,
9586    last_value: Option<(GaugeVerdict, Duration)>,
9587}
9588
9589/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9590/// its ceiling can say in its timeout line which declared gauge held it and
9591/// why. Without it, a gauge that read busy, a gauge the module never
9592/// reported, and a probe that never got an answer all looked the same: a
9593/// drain that ended at its ceiling with nothing in flight.
9594#[derive(Debug, Clone, Default, PartialEq, Eq)]
9595struct DrainGaugeTrace {
9596    /// Health probes the drain sent.
9597    probes: u32,
9598    /// Each declared gauge, in declaration order.
9599    gauges: Vec<(String, GaugeTrace)>,
9600}
9601
9602impl DrainGaugeTrace {
9603    fn new(gauges: &[String]) -> Self {
9604        Self {
9605            probes: 0,
9606            gauges: gauges
9607                .iter()
9608                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9609                .collect(),
9610        }
9611    }
9612
9613    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9614    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9615        self.probes = self.probes.saturating_add(1);
9616        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9617            trace.latest = Some((verdict.clone(), at));
9618            if matches!(
9619                verdict,
9620                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9621            ) {
9622                trace.last_value = Some((verdict.clone(), at));
9623            }
9624        }
9625    }
9626}
9627
9628/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
9629/// declared gauge, separated by ` | `. A value is shown as its verdict
9630/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
9631/// and whether that counted as busy.
9632impl fmt::Display for DrainGaugeTrace {
9633    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9634        if self.gauges.is_empty() {
9635            return f.write_str("none declared");
9636        }
9637        for (index, (name, trace)) in self.gauges.iter().enumerate() {
9638            if index > 0 {
9639                f.write_str(" | ")?;
9640            }
9641            write!(f, "{name}: ")?;
9642            match &trace.latest {
9643                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9644                None => f.write_str("never probed")?,
9645            }
9646            match &trace.last_value {
9647                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9648                None => f.write_str("; no value ever read")?,
9649            }
9650        }
9651        Ok(())
9652    }
9653}
9654
9655fn declared_busy_gauges(
9656    registry: &Registry,
9657    module_id: &str,
9658) -> Result<Vec<String>, SuperviseError> {
9659    busy_gauges_of(
9660        registry
9661            .get_module(module_id)
9662            .map_err(SuperviseError::Registry)?,
9663    )
9664}
9665
9666/// [`declared_busy_gauges`] for the registration a connection holds, in any
9667/// slot: after cutover the incumbent is no longer the id's active
9668/// registration, and its own manifest is the one that names its gauges.
9669fn declared_busy_gauges_for_connection(
9670    registry: &Registry,
9671    connection_id: ConnectionId,
9672) -> Result<Vec<String>, SuperviseError> {
9673    busy_gauges_of(
9674        registry
9675            .get_module_by_connection(connection_id)
9676            .map_err(SuperviseError::Registry)?,
9677    )
9678}
9679
9680fn busy_gauges_of(
9681    registration: Option<crate::registry::ModuleRegistration>,
9682) -> Result<Vec<String>, SuperviseError> {
9683    let Some(registration) = registration else {
9684        return Ok(Vec::new());
9685    };
9686    let Some(self_signals) = registration.manifest.self_signals else {
9687        return Ok(Vec::new());
9688    };
9689
9690    let mut gauges = Vec::new();
9691    for declaration in self_signals {
9692        if declaration.kind != SelfSignalKind::Busy {
9693            continue;
9694        }
9695        match declaration.anchored_to {
9696            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9697                gauges.extend(declared)
9698            }
9699            _ => {
9700                // An invalid Busy anchor is fail-safe: the empty name cannot be
9701                // present in a conforming health report, so this drain stays busy.
9702                gauges.push(String::new());
9703            }
9704        }
9705    }
9706    Ok(gauges)
9707}
9708
9709/// Wait for `endpoint` to have nothing in flight and, when the module declares
9710/// busy gauges, for a health probe to report them quiet. The probe is addressed
9711/// by `scope`: a swap's superseded incumbent must be asked about its own
9712/// gauges, and by module id the probe would reach the promoted candidate.
9713/// While the gauges read busy, the probe repeats every
9714/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9715/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
9716/// timeout line.
9717#[allow(clippy::too_many_arguments)]
9718async fn wait_for_forwarding_quiescence(
9719    forwarding: &ForwardingTable,
9720    module_id: &str,
9721    runtime: &SupervisorRuntimeConfig,
9722    endpoint: crate::ModuleEndpointId,
9723    deadline: Instant,
9724    busy_gauges: &[String],
9725    scope: DrainScope,
9726    trace: &mut DrainGaugeTrace,
9727) -> Result<bool, SuperviseError> {
9728    let wait_started = Instant::now();
9729    let mut gauges_quiescent = busy_gauges.is_empty();
9730    let mut next_probe_at = wait_started;
9731    let mut omission_reported = false;
9732
9733    loop {
9734        let now = Instant::now();
9735        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9736            let report = match scope {
9737                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9738                DrainScope::Endpoint(endpoint) => {
9739                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9740                }
9741            };
9742            let verdicts = match &report {
9743                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9744                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9745            };
9746            trace.record(&verdicts, wait_started.elapsed());
9747            gauges_quiescent = match report {
9748                Ok(_) => match busy_gauge_observation(&verdicts) {
9749                    BusyGaugeObservation::Quiescent => true,
9750                    BusyGaugeObservation::Busy => false,
9751                    BusyGaugeObservation::Omitted => {
9752                        // Once per drain: a module that never reports a gauge
9753                        // it declared would otherwise say so on every probe.
9754                        if !omission_reported {
9755                            forwarding
9756                                .counters()
9757                                .increment_drains_with_undeclared_gauge();
9758                            let missing = busy_gauges
9759                                .iter()
9760                                .zip(&verdicts)
9761                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9762                                .map(|(gauge, _)| gauge.as_str())
9763                                .collect::<Vec<_>>()
9764                                .join(",");
9765                            warn!(
9766                                module_id,
9767                                missing_gauges = %missing,
9768                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9769                            );
9770                            omission_reported = true;
9771                        }
9772                        false
9773                    }
9774                },
9775                Err(err) => {
9776                    warn!(
9777                        module_id,
9778                        error = %err,
9779                        "drain health.check did not produce declared busy gauges; treating module as busy"
9780                    );
9781                    false
9782                }
9783            };
9784            // Counted from the probe's completion: the probe above was awaited,
9785            // so no second probe can start while one is still outstanding.
9786            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9787        }
9788
9789        let in_flight = forwarding
9790            .endpoint_in_flight_count(endpoint)
9791            .map_err(SuperviseError::Forwarding)?;
9792        if in_flight == 0 && gauges_quiescent {
9793            return Ok(true);
9794        }
9795
9796        let now = Instant::now();
9797        if now >= deadline {
9798            return Ok(false);
9799        }
9800        let mut wait = deadline
9801            .saturating_duration_since(now)
9802            .min(REGISTRY_RELEASE_POLL);
9803        if !busy_gauges.is_empty() {
9804            wait = wait.min(next_probe_at.saturating_duration_since(now));
9805        }
9806        sleep(wait).await;
9807    }
9808}
9809
9810/// The wait between drain busy-gauge probes: the larger of
9811/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
9812/// drain's own in-flight poll interval.
9813fn drain_busy_reprobe_interval() -> Duration {
9814    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9815}
9816
9817/// Logs why a drain gave up: the drain waited its full timeout without the
9818/// module going quiet, and teardown is being forced.
9819///
9820/// The line names two kinds of evidence, because a bare "timed out" hides
9821/// causes that need different fixes:
9822/// - `holdouts`: the requests and routes still open when the timeout hit,
9823///   and on which connections. One stuck call is a module bug; a held stream
9824///   on every session is a module that should end its streams when it
9825///   receives `module.draining`.
9826/// - `gauges`: for each busy gauge the module declares, its latest reading
9827///   and how many probes ran. This separates a module that kept reporting
9828///   itself busy, one whose health answer never contained the gauge, and one
9829///   that never answered the probes, which otherwise all look like a timeout
9830///   with nothing in flight.
9831fn warn_drain_timed_out(
9832    module_id: &str,
9833    drain_timeout: Duration,
9834    reason: RouteCloseReason,
9835    holdouts: &crate::forwarding::DrainHoldouts,
9836    gauges: &DrainGaugeTrace,
9837) {
9838    warn!(
9839        module_id = %module_id,
9840        waited = ?drain_timeout,
9841        ?reason,
9842        held_requests = holdouts.requests,
9843        held_routes = holdouts.routes,
9844        total_routes = holdouts.total_routes,
9845        top_connections = ?holdouts.top_connections,
9846        // `module_channel:corr`, so the module can find each held request
9847        // in its own log; capped, so `held_requests` is the full count.
9848        held = %holdouts
9849            .held
9850            .iter()
9851            .map(|(channel, corr)| format!("{channel}:{corr}"))
9852            .collect::<Vec<_>>()
9853            .join(","),
9854        gauge_probes = gauges.probes,
9855        busy_gauges = %gauges,
9856        "route drain timed out before request quiescence; forcing teardown"
9857    );
9858}
9859
9860/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9861///
9862/// `Ok` is always honest and passed straight through -- the wait actually measured
9863/// in-flight state. `Err` means the wait produced no measurement at all (the
9864/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9865/// constant: the drain did not complete. Never recomputed from route state, never a
9866/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9867fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9868    match wait_result {
9869        Ok(drained) => *drained,
9870        Err(_) => false,
9871    }
9872}
9873
9874fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9875    for released in released_routes {
9876        let frame = match Frame::build_with_version(
9877            released.negotiated_ver,
9878            FrameType::Goodbye,
9879            control_flags(),
9880            released.channel,
9881            released.epoch,
9882            0,
9883            Vec::new(),
9884        ) {
9885            Ok(frame) => frame,
9886            Err(err) => {
9887                warn!(
9888                    route_channel = released.channel,
9889                    error = %err,
9890                    "failed to build supervisor drain route GOODBYE frame"
9891                );
9892                continue;
9893            }
9894        };
9895        if !released.close_on_delivery_failure() {
9896            crate::forwarding::send_module_route_goodbye(
9897                &forwarding.counters(),
9898                &released.sink,
9899                frame,
9900                released.module_id.as_deref(),
9901                "supervisor drain",
9902            );
9903            continue;
9904        }
9905        if let Err(err) = released.sink.try_send(frame) {
9906            warn!(
9907                target_connection_id = released.connection_id.get(),
9908                route_channel = released.channel,
9909                error = %err,
9910                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9911            );
9912            let _ = forwarding.escalate_client_delivery_failure(
9913                released.connection_id,
9914                released.channel,
9915                released.epoch,
9916                CloseReason::new(
9917                    "route_goodbye_delivery_failed",
9918                    format!(
9919                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9920                        released.channel
9921                    ),
9922                ),
9923                crate::forwarding::UndeliveredFrame {
9924                    module_id: released.module_id.as_deref(),
9925                    sink: &released.sink,
9926                },
9927            );
9928        }
9929    }
9930}
9931
9932fn send_module_draining(
9933    module_id: &str,
9934    reason: RouteCloseReason,
9935    deadline_ms: u64,
9936    target: &ModuleDrainTarget,
9937) {
9938    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9939        reason,
9940        deadline_ms,
9941    }) {
9942        Ok(body) => body,
9943        Err(err) => {
9944            warn!(
9945                module_id,
9946                error = %err,
9947                "failed to encode module draining command"
9948            );
9949            return;
9950        }
9951    };
9952    let frame = match Frame::build_with_version(
9953        target.negotiated_ver,
9954        FrameType::Push,
9955        control_flags(),
9956        0,
9957        0,
9958        0,
9959        body,
9960    ) {
9961        Ok(frame) => frame,
9962        Err(err) => {
9963            warn!(
9964                module_id,
9965                error = %err,
9966                "failed to build module draining command frame"
9967            );
9968            return;
9969        }
9970    };
9971    if let Err(err) = target.sink.try_send(frame) {
9972        warn!(
9973            module_id,
9974            target_connection_id = target.endpoint.connection_id.get(),
9975            error = %err,
9976            "module draining command was not delivered to peer"
9977        );
9978    }
9979}
9980
9981/// The channel-0 GOODBYE that tells a module its stop is planned.
9982fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9983    match Frame::build_with_version(
9984        negotiated_ver,
9985        FrameType::Goodbye,
9986        control_flags(),
9987        0,
9988        0,
9989        0,
9990        Vec::new(),
9991    ) {
9992        Ok(frame) => Some(frame),
9993        Err(err) => {
9994            warn!(
9995                module_id,
9996                error = %err,
9997                "failed to build module GOODBYE frame"
9998            );
9999            None
10000        }
10001    }
10002}
10003
10004/// Send every registered module connection its module GOODBYE at daemon
10005/// shutdown, then request that connection's close.
10006///
10007/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10008/// before EOF, so the GOODBYE must reach the socket before the close. A close
10009/// request does not wait for the connection's queued frames: its writer gets a
10010/// bounded grace after the close, is aborted if it overruns it, and the daemon
10011/// process may exit before that grace ends. So with `wait_for_flush`, each
10012/// connection is closed only after its writer has acknowledged writing the
10013/// GOODBYE, or once a short shared budget runs out, so one module that is not
10014/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10015/// are only queued, for a shutdown the operator has told to stop waiting.
10016/// A connection that is already gone is skipped.
10017#[cfg(unix)]
10018async fn send_module_goodbyes_for_daemon_shutdown(
10019    forwarding: &Arc<ForwardingTable>,
10020    reason: &CloseReason,
10021    wait_for_flush: bool,
10022) {
10023    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10024    let targets = match forwarding.module_connections() {
10025        Ok(targets) => targets,
10026        Err(err) => {
10027            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10028            return;
10029        }
10030    };
10031    let deadline = Instant::now() + GOODBYE_BUDGET;
10032    let mut sends = tokio::task::JoinSet::new();
10033    for target in targets {
10034        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10035            continue;
10036        };
10037        if !wait_for_flush {
10038            if let Err(err) = target.sink.try_send(frame) {
10039                debug!(
10040                    module_id = %target.module_id,
10041                    error = %err,
10042                    "shutdown module GOODBYE was not queued"
10043                );
10044            }
10045            continue;
10046        }
10047        let forwarding = Arc::clone(forwarding);
10048        let reason = reason.clone();
10049        sends.spawn(async move {
10050            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10051                Ok(Ok(())) => {}
10052                Ok(Err(err)) => debug!(
10053                    module_id = %target.module_id,
10054                    error = %err,
10055                    "module connection closed before its shutdown GOODBYE was written"
10056                ),
10057                Err(_) => warn!(
10058                    module_id = %target.module_id,
10059                    budget = ?GOODBYE_BUDGET,
10060                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10061                ),
10062            }
10063            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10064        });
10065    }
10066    // Every task ends by the shared deadline, so this wait is bounded too.
10067    while sends.join_next().await.is_some() {}
10068}
10069
10070fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10071    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10072        return;
10073    };
10074    if let Err(err) = target.sink.try_send(frame) {
10075        warn!(
10076            module_id,
10077            target_connection_id = target.endpoint.connection_id.get(),
10078            error = %err,
10079            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10080        );
10081        forwarding.request_connection_close(
10082            target.endpoint.connection_id,
10083            CloseReason::new(
10084                "module_goodbye_delivery_failed",
10085                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10086            ),
10087        );
10088    }
10089}
10090
10091#[derive(Clone, Copy)]
10092struct ForwardingDrainContext<'a> {
10093    spec: &'a ModuleSpec,
10094    runtime: &'a SupervisorRuntimeConfig,
10095    registry: &'a Registry,
10096    scope: DrainScope,
10097}
10098
10099/// Which process a forwarding drain addresses.
10100#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10101enum DrainScope {
10102    /// Whatever endpoint is active for the module id: every plain stop,
10103    /// restart and reload. Also moves the module's state to `Draining`.
10104    Active,
10105    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10106    /// module id would resolve to the promoted candidate and leave neither
10107    /// process routable. The module's state is left alone, since the promoted
10108    /// candidate is what it describes and that process is running.
10109    Endpoint(crate::ModuleEndpointId),
10110}
10111
10112/// Whether a child being drained has already been asked to stop by the time
10113/// its drain wait starts.
10114///
10115/// The drain wait is the same budget whatever this says. What it decides is
10116/// whether the supervisor must ask by signal before that wait begins: a child
10117/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10118/// healthy or not.
10119#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10120enum StopNotice {
10121    /// The module was sent `module.draining` and a module GOODBYE over its own
10122    /// registered connection, and stops itself.
10123    SentOverConnection,
10124    /// The forwarding drain found no registered connection for the module: a
10125    /// subc child spawned moments ago that has not sent HELLO yet, or a
10126    /// `protocol: "none"` child, which never registers.
10127    NoConnection,
10128    /// This path sends nothing over the module's connection: the supervisor has
10129    /// no forwarding table, or the caller stops the child without a forwarding
10130    /// drain.
10131    NotSent,
10132}
10133
10134async fn begin_forwarding_drain(
10135    spec: &ModuleSpec,
10136    runtime: &SupervisorRuntimeConfig,
10137    registry: &Registry,
10138    snapshot: &SharedSnapshot,
10139    enabled: Option<bool>,
10140    reason: RouteCloseReason,
10141) -> Result<StopNotice, SuperviseError> {
10142    let Some(forwarding) = runtime.forwarding.as_ref() else {
10143        return Err(SuperviseError::ReloadUnavailable {
10144            module_id: spec.module_id.clone(),
10145            reason: "supervisor was not configured with a forwarding table".to_string(),
10146        });
10147    };
10148
10149    begin_forwarding_drain_with(
10150        forwarding,
10151        ForwardingDrainContext {
10152            spec,
10153            runtime,
10154            registry,
10155            scope: DrainScope::Active,
10156        },
10157        snapshot,
10158        enabled,
10159        reason,
10160        runtime.drain_timeout,
10161    )
10162    .await
10163}
10164
10165async fn begin_forwarding_drain_if_configured(
10166    spec: &ModuleSpec,
10167    runtime: &SupervisorRuntimeConfig,
10168    registry: &Registry,
10169    snapshot: &SharedSnapshot,
10170    enabled: Option<bool>,
10171    reason: RouteCloseReason,
10172) -> Result<StopNotice, SuperviseError> {
10173    begin_forwarding_drain_with_timeout(
10174        spec,
10175        runtime,
10176        registry,
10177        snapshot,
10178        enabled,
10179        reason,
10180        runtime.drain_timeout,
10181    )
10182    .await
10183}
10184
10185/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10186/// budget, for paths where the operator overrides the module's configured one
10187/// (`supervisor.restart{drain_timeout_ms}`).
10188async fn begin_forwarding_drain_with_timeout(
10189    spec: &ModuleSpec,
10190    runtime: &SupervisorRuntimeConfig,
10191    registry: &Registry,
10192    snapshot: &SharedSnapshot,
10193    enabled: Option<bool>,
10194    reason: RouteCloseReason,
10195    drain_timeout: Duration,
10196) -> Result<StopNotice, SuperviseError> {
10197    let Some(forwarding) = runtime.forwarding.as_ref() else {
10198        return Ok(StopNotice::NotSent);
10199    };
10200
10201    begin_forwarding_drain_with(
10202        forwarding,
10203        ForwardingDrainContext {
10204            spec,
10205            runtime,
10206            registry,
10207            scope: DrainScope::Active,
10208        },
10209        snapshot,
10210        enabled,
10211        reason,
10212        drain_timeout,
10213    )
10214    .await
10215}
10216
10217async fn begin_forwarding_drain_with(
10218    forwarding: &ForwardingTable,
10219    context: ForwardingDrainContext<'_>,
10220    snapshot: &SharedSnapshot,
10221    enabled: Option<bool>,
10222    reason: RouteCloseReason,
10223    drain_timeout: Duration,
10224) -> Result<StopNotice, SuperviseError> {
10225    let ForwardingDrainContext {
10226        spec,
10227        runtime,
10228        registry,
10229        scope,
10230    } = context;
10231    debug_assert_ne!(reason, RouteCloseReason::Crash);
10232    let terminal = matches!(reason, RouteCloseReason::Disable);
10233    let drain_started_at = Instant::now();
10234    let drain_deadline = drain_started_at + drain_timeout;
10235    let deadline_ms =
10236        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10237    let busy_gauges = match scope {
10238        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10239        DrainScope::Endpoint(endpoint) => {
10240            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10241        }
10242    };
10243
10244    // Admission gate first: route.open/commit and route REQUEST admission are closed
10245    // before the first quiescence check, so the outstanding count can only fall.
10246    let gate_started = Instant::now();
10247    let drain_target = match scope {
10248        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10249        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10250    }
10251    .map_err(SuperviseError::Forwarding)?;
10252    // The instant admission closed, and how long taking the forwarding write
10253    // lock to close it took. The timeout line reports only the quiescence
10254    // wait, so without this a drain that started late looked like one that
10255    // started on time.
10256    info!(
10257        module_id = %spec.module_id,
10258        ?reason,
10259        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10260        connected = drain_target.is_some(),
10261        "module drain began; route admission closed"
10262    );
10263    if scope == DrainScope::Active {
10264        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10265            state.state = ModuleState::Draining;
10266            state.draining_to_replace =
10267                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10268            if let Some(enabled) = enabled {
10269                state.enabled = enabled;
10270            }
10271        })?;
10272    }
10273
10274    let Some(target) = drain_target.as_ref() else {
10275        // Nothing was sent: the module has no registered connection to carry
10276        // `module.draining` or a GOODBYE. The caller must not assume the child
10277        // was asked to stop.
10278        return Ok(StopNotice::NoConnection);
10279    };
10280    {
10281        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10282        #[cfg(feature = "test-support")]
10283        forwarding.wait_before_route_closing_for_test().await;
10284        let routes = forwarding
10285            .endpoint_routes(target.endpoint)
10286            .map_err(SuperviseError::Forwarding)?;
10287        let routes_notified = routes.len();
10288        crate::control::send_route_control_pushes(
10289            forwarding,
10290            routes.clone(),
10291            ClientControlPush::RouteClosing {
10292                module_id: spec.module_id.clone(),
10293                channels: Vec::new(),
10294                reason,
10295            },
10296        );
10297        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10298
10299        // `route.closing` was just sent above: from here on every return path,
10300        // including an early one, MUST send `route.closed` before propagating
10301        // anything else. A client holds `closing` as a promise that a verdict is
10302        // coming; leaving early without `closed` strands it waiting forever, since
10303        // `closing` carries no timeout of its own.
10304        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10305        let wait_result = wait_for_forwarding_quiescence(
10306            forwarding,
10307            &spec.module_id,
10308            runtime,
10309            target.endpoint,
10310            drain_deadline,
10311            &busy_gauges,
10312            scope,
10313            &mut gauge_trace,
10314        )
10315        .await;
10316        let drained = drained_after_quiescence_wait(&wait_result);
10317        if let Err(err) = &wait_result {
10318            error!(
10319                module_id = %spec.module_id,
10320                ?reason,
10321                error = %err,
10322                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10323            );
10324        } else if !drained {
10325            // Read before teardown releases the routes.
10326            let holdouts = forwarding
10327                .endpoint_drain_holdouts(target.endpoint)
10328                .unwrap_or_default();
10329            warn_drain_timed_out(
10330                &spec.module_id,
10331                drain_timeout,
10332                reason,
10333                &holdouts,
10334                &gauge_trace,
10335            );
10336        }
10337        crate::control::send_route_control_pushes(
10338            forwarding,
10339            routes,
10340            ClientControlPush::RouteClosed {
10341                module_id: spec.module_id.clone(),
10342                channels: Vec::new(),
10343                reason,
10344                drained,
10345                abandoned: target.abandoned_bindings.len() as u32,
10346                excluded_subscriptions: target.excluded_subscriptions,
10347                terminal: Some(terminal),
10348            },
10349        );
10350        wait_result?;
10351
10352        // `route.closed` has now been sent unconditionally above. From here the
10353        // remaining steps are cleanup (route + module GOODBYE) rather than a
10354        // promise the client is waiting on, but a lock-poisoned
10355        // `release_module_endpoint_routes` would otherwise skip the module
10356        // GOODBYE silently too -- send it before propagating the error.
10357        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10358            Ok(routes) => routes,
10359            Err(err) => {
10360                warn!(
10361                    module_id = %spec.module_id,
10362                    ?reason,
10363                    error = %err,
10364                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10365                );
10366                send_module_goodbye(&spec.module_id, forwarding, target);
10367                return Err(SuperviseError::Forwarding(err));
10368            }
10369        };
10370        let route_goodbye_count = released_routes.len();
10371        send_route_goodbyes(forwarding, released_routes);
10372        send_module_goodbye(&spec.module_id, forwarding, target);
10373
10374        // The drain's happy path was previously silent: every emission above is
10375        // best-effort with only its failure arm logged, so "were consumers told"
10376        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10377        // hang where the open question was exactly whether teardown notice went
10378        // out). One summary line makes that class decidable in one grep.
10379        info!(
10380            module_id = %spec.module_id,
10381            ?reason,
10382            routes_notified,
10383            route_goodbyes = route_goodbye_count,
10384            abandoned_reservations = target.abandoned_bindings.len(),
10385            excluded_subscriptions = target.excluded_subscriptions,
10386            drained,
10387            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10388        );
10389    }
10390
10391    Ok(StopNotice::SentOverConnection)
10392}
10393
10394/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10395/// the only slot a plain (non-swap) spawn can register into.
10396async fn wait_for_registration_after_reload(
10397    registry: &Registry,
10398    module_id: &str,
10399    snapshot: &SharedSnapshot,
10400    child: &mut SupervisedChild,
10401    wait: Duration,
10402) -> Result<RegistrationWaitOutcome, SuperviseError> {
10403    wait_for_slot_registration(
10404        registry,
10405        crate::registry::RegistrationSlot::Active(module_id),
10406        module_id,
10407        snapshot,
10408        child,
10409        wait,
10410    )
10411    .await
10412}
10413
10414/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10415///
10416/// Keyed on the slot rather than the bare module id because during a swap the
10417/// id's active slot is already held by the incumbent: an id-keyed wait would
10418/// report the incumbent's registration as the candidate's and a candidate that
10419/// never registers would look registered. A swap candidate waits on
10420/// `crate::registry::RegistrationSlot::Candidate`.
10421async fn wait_for_slot_registration(
10422    registry: &Registry,
10423    slot: crate::registry::RegistrationSlot<'_>,
10424    module_id: &str,
10425    snapshot: &SharedSnapshot,
10426    child: &mut SupervisedChild,
10427    wait: Duration,
10428) -> Result<RegistrationWaitOutcome, SuperviseError> {
10429    let deadline = Instant::now() + wait;
10430    loop {
10431        if registry
10432            .registration(slot)
10433            .map_err(SuperviseError::Registry)?
10434            .is_some()
10435        {
10436            return Ok(RegistrationWaitOutcome::Registered);
10437        }
10438
10439        let now = Instant::now();
10440        if now >= deadline {
10441            return Ok(RegistrationWaitOutcome::TimedOut);
10442        }
10443        let remaining = deadline.saturating_duration_since(now);
10444        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10445
10446        tokio::select! {
10447            wait_result = child.wait() => {
10448                let status = wait_result.map_err(|source| SuperviseError::Wait {
10449                    module_id: module_id.to_string(),
10450                    source,
10451                })?;
10452                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10453                    snapshot,
10454                    child,
10455                    &status,
10456                )));
10457            }
10458            _ = sleep(poll) => {}
10459        }
10460    }
10461}
10462
10463fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10464    // A replacement process that exits before HELLO did not provide service, even
10465    // if it used status 0. Count it against the restart cap as a new-binary failure.
10466    if exit_report.kind != ExitKind::DeliberateSeverance {
10467        exit_report.kind = ExitKind::Crash;
10468    }
10469    exit_report
10470}
10471
10472async fn handle_reload_child_registration_failure(
10473    spec: &ModuleSpec,
10474    runtime: &SupervisorRuntimeConfig,
10475    registry: &Registry,
10476    process_liveness: &SupervisorProcessLiveness,
10477    snapshot: &SharedSnapshot,
10478    _child: &mut Option<SupervisedChild>,
10479    failure: ReloadRegistrationFailure,
10480) -> Result<(), SuperviseError> {
10481    let ReloadRegistrationFailure {
10482        exit_report,
10483        reason,
10484    } = failure;
10485    match on_child_exit(
10486        spec,
10487        runtime.restart_policy,
10488        registry,
10489        snapshot,
10490        &runtime.terminal_ring,
10491        &runtime.spawn_events,
10492        &runtime.child_roster,
10493        exit_report,
10494    )
10495    .await
10496    {
10497        NextAction::Stop {
10498            registration_released,
10499        } => {
10500            if registration_released {
10501                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10502            }
10503        }
10504        NextAction::Restart { schedule } => {
10505            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10506                schedule.delay
10507            });
10508            if let Some(schedule) = schedule {
10509                log_crash_respawn(&spec.module_id, schedule);
10510            }
10511            schedule_respawn(
10512                runtime,
10513                snapshot,
10514                &spec.module_id,
10515                delay,
10516                RespawnKind::Spawn,
10517            )?;
10518        }
10519    }
10520    Err(SuperviseError::ReloadFailed {
10521        module_id: spec.module_id.clone(),
10522        reason,
10523    })
10524}
10525
10526async fn handle_reload_spawn_failure(
10527    spec: &ModuleSpec,
10528    runtime: &SupervisorRuntimeConfig,
10529    process_liveness: &SupervisorProcessLiveness,
10530    snapshot: &SharedSnapshot,
10531    _child: &mut Option<SupervisedChild>,
10532    reason: String,
10533) -> Result<(), SuperviseError> {
10534    let now = Instant::now();
10535    let mut schedule = None;
10536    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10537        clear_current_process_facts(state);
10538        if state.enabled {
10539            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10540            state.state = if schedule.is_some() {
10541                ModuleState::Restarting
10542            } else {
10543                ModuleState::Failed
10544            };
10545        } else {
10546            state.state = ModuleState::Disabled;
10547        }
10548    })?;
10549    if let Some(schedule) = schedule {
10550        schedule_respawn(
10551            runtime,
10552            snapshot,
10553            &spec.module_id,
10554            schedule.delay,
10555            RespawnKind::Spawn,
10556        )?;
10557    } else {
10558        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10559    }
10560    Err(SuperviseError::ReloadFailed {
10561        module_id: spec.module_id.clone(),
10562        reason,
10563    })
10564}
10565
10566fn control_flags() -> Flags {
10567    Flags::new(false, Priority::Passive, false)
10568}
10569
10570#[allow(clippy::too_many_arguments)]
10571async fn drain_optional_child(
10572    module_id: &str,
10573    protocol: ModuleProtocol,
10574    stop_notice: StopNotice,
10575    registry: &Registry,
10576    forwarding: Option<&ForwardingTable>,
10577    snapshot: &SharedSnapshot,
10578    terminal_ring: &Arc<Mutex<TerminalRing>>,
10579    spawn_events: &SpawnEventFeed,
10580    child: &mut Option<SupervisedChild>,
10581    drain_timeout: Duration,
10582    final_state: ModuleState,
10583    enabled: Option<bool>,
10584) -> Result<(), SuperviseError> {
10585    if let Some(child) = child.take() {
10586        drain_child_to_state(
10587            module_id,
10588            protocol,
10589            stop_notice,
10590            registry,
10591            forwarding,
10592            snapshot,
10593            terminal_ring,
10594            spawn_events,
10595            child,
10596            drain_timeout,
10597            final_state,
10598            enabled,
10599        )
10600        .await
10601    } else {
10602        update_snapshot(snapshot, Some(module_id), |state| {
10603            state.state = final_state;
10604            if let Some(enabled) = enabled {
10605                state.enabled = enabled;
10606            }
10607            clear_current_process_facts(state);
10608        })?;
10609        release_dead_registration(registry, forwarding, snapshot, module_id).await
10610    }
10611}
10612
10613#[allow(clippy::too_many_arguments)]
10614async fn drain_child_to_state(
10615    module_id: &str,
10616    _protocol: ModuleProtocol,
10617    stop_notice: StopNotice,
10618    registry: &Registry,
10619    forwarding: Option<&ForwardingTable>,
10620    snapshot: &SharedSnapshot,
10621    terminal_ring: &Arc<Mutex<TerminalRing>>,
10622    spawn_events: &SpawnEventFeed,
10623    mut child: SupervisedChild,
10624    drain_timeout: Duration,
10625    final_state: ModuleState,
10626    enabled: Option<bool>,
10627) -> Result<(), SuperviseError> {
10628    let protocol = child.protocol;
10629    update_snapshot(snapshot, Some(module_id), |state| {
10630        state.state = ModuleState::Draining;
10631        state.draining_to_replace = final_state == ModuleState::Restarting;
10632        if let Some(enabled) = enabled {
10633            state.enabled = enabled;
10634        }
10635    })?;
10636
10637    // The wait below is the same budget in every case; what differs is
10638    // whether anything has ASKED the child to stop before it starts. Only a
10639    // forwarding drain that reached the module's registered connection has
10640    // (`module.draining`, then a module GOODBYE). Every other child was told
10641    // nothing: a `protocol: "none"` module, which never registers; a subc
10642    // module spawned moments ago that has not sent HELLO yet; or a stop that
10643    // runs no forwarding drain. Without a signal the budget is only a delay
10644    // in front of SIGKILL -- and the not-yet-registered child is the worst
10645    // case, because it registers into a module that is already draining,
10646    // is never told, and is killed while healthy.
10647    if stop_notice != StopNotice::SentOverConnection {
10648        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10649            info!(
10650                module_id,
10651                pid = child.pid,
10652                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10653                "module has no connection yet; requesting stop by signal"
10654            );
10655        }
10656        request_graceful_stop(module_id, &child);
10657    }
10658
10659    let exit_report = match timeout(drain_timeout, child.wait()).await {
10660        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10661        Ok(Err(source)) => {
10662            fail_after_wait_error(
10663                module_id,
10664                snapshot,
10665                terminal_ring,
10666                spawn_events,
10667                format!("failed to wait during drain: {source}"),
10668            );
10669            return Err(SuperviseError::Wait {
10670                module_id: module_id.to_string(),
10671                source,
10672            });
10673        }
10674        Err(_) => {
10675            // Mirror the sibling arm above: state is already `Draining`, and an
10676            // error propagated from here would strand it there -- a state
10677            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10678            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10679            // `Failed` before `?` keeps the module operator-visible and
10680            // revivable. Trigger is an ESRCH race (process exits between the
10681            // drain timeout firing and the kill) or a post-kill wait failure
10682            // (issue #34).
10683            //
10684            // Logged because the kill is otherwise visible only as signal 9 in
10685            // the terminal ring, and the budget it follows can be long enough
10686            // that consumers see a stretch of refusals with no stated cause.
10687            warn!(
10688                module_id,
10689                pid = child.pid,
10690                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10691                reason = ?final_state,
10692                ?stop_notice,
10693                "drain budget expired before the module exited; killing it"
10694            );
10695            child.start_kill().map_err(|source| {
10696                fail_after_wait_error(
10697                    module_id,
10698                    snapshot,
10699                    terminal_ring,
10700                    spawn_events,
10701                    format!("failed to kill during drain: {source}"),
10702                );
10703                SuperviseError::Kill {
10704                    module_id: module_id.to_string(),
10705                    source,
10706                }
10707            })?;
10708            let status = child.wait().await.map_err(|source| {
10709                fail_after_wait_error(
10710                    module_id,
10711                    snapshot,
10712                    terminal_ring,
10713                    spawn_events,
10714                    format!("failed to wait after drain kill: {source}"),
10715                );
10716                SuperviseError::Wait {
10717                    module_id: module_id.to_string(),
10718                    source,
10719                }
10720            })?;
10721            classify_reaped_child_exit(snapshot, &child, &status)
10722        }
10723    };
10724
10725    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10726    publish_after_terminal_record(
10727        module_id,
10728        || {
10729            record_terminal_with_detail(
10730                module_id,
10731                terminal_ring,
10732                spawn_events,
10733                &exit_report,
10734                terminal_disposition(final_state),
10735                detail,
10736            )
10737        },
10738        || {
10739            update_snapshot(snapshot, Some(module_id), |state| {
10740                state.state = final_state;
10741                if let Some(enabled) = enabled {
10742                    state.enabled = enabled;
10743                }
10744                clear_current_process_facts(state);
10745                state.last_exit = Some(exit_report.clone());
10746                if exit_report.kind == ExitKind::DeliberateSeverance {
10747                    state.lifetime_restarts += 1;
10748                }
10749            })
10750        },
10751    )?;
10752    child.drain_stderr(module_id).await;
10753
10754    release_dead_registration(registry, forwarding, snapshot, module_id).await
10755}
10756
10757/// Ask a child that nothing else has asked to stop, by signal.
10758///
10759/// A registered subc module is asked over its own connection: the drain sends
10760/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10761/// module GOODBYE, and the module stops itself. A module that speaks no subc
10762/// wire receives none of that, and neither does a subc module that has not
10763/// registered yet, so for them the drain budget would be pure delay in front of
10764/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10765/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10766/// into a recovery on the next start.
10767///
10768/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10769/// rule rather than an optimisation: that module's graceful stop is already
10770/// running by the time its child is drained, and a signal would race it.
10771///
10772/// Best-effort by construction. A child that has already exited is the ordinary
10773/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10774/// failure is logged at debug and the wait-then-kill below still decides the
10775/// outcome.
10776#[cfg(unix)]
10777fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10778    let Some(pid) = child
10779        .id()
10780        .and_then(|pid| i32::try_from(pid).ok())
10781        .and_then(rustix::process::Pid::from_raw)
10782    else {
10783        debug!(
10784            module_id,
10785            "no pid to signal for teardown; falling through to the drain wait"
10786        );
10787        return;
10788    };
10789    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10790        Ok(()) => debug!(
10791            module_id,
10792            "sent SIGTERM to a module nothing else asked to stop"
10793        ),
10794        Err(err) => debug!(
10795            module_id,
10796            error = %err,
10797            "SIGTERM to module failed; the drain wait and kill still apply"
10798        ),
10799    }
10800}
10801
10802/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10803/// Windows does offer need cooperation this supervisor cannot assume: a console
10804/// control event requires sharing a console with the child, and `WM_CLOSE`
10805/// requires the child to pump a message loop. A supervised server process does
10806/// neither, so there is nothing to send and teardown is the wait followed by the
10807/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10808/// the thing `protocol: "none"` exists to avoid.
10809#[cfg(not(unix))]
10810fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10811    debug!(
10812        module_id,
10813        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10814    );
10815}
10816
10817fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10818    match final_state {
10819        ModuleState::Stopped => TerminalDisposition::Stopped,
10820        ModuleState::Disabled => TerminalDisposition::Disabled,
10821        ModuleState::Restarting => TerminalDisposition::Restarting,
10822        ModuleState::Failed => TerminalDisposition::Failed,
10823        ModuleState::Starting
10824        | ModuleState::Running
10825        | ModuleState::Unresponsive
10826        | ModuleState::Draining => {
10827            unreachable!("terminal exits only finish in terminal or restarting states")
10828        }
10829    }
10830}
10831
10832/// Release a reaped child's registration before allowing another spawn.
10833///
10834/// EOF is not a process-lifetime signal: an inherited socket can stay open
10835/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10836/// reading EOF. After the normal release grace, request connection close (which
10837/// cancels both reads and dispatch), then allow one more release grace for the
10838/// connection guard's forwarding cleanup. Never evict a different connection.
10839async fn release_dead_registration(
10840    registry: &Registry,
10841    forwarding: Option<&ForwardingTable>,
10842    snapshot: &SharedSnapshot,
10843    module_id: &str,
10844) -> Result<(), SuperviseError> {
10845    let result = async {
10846        let registration = registry
10847            .get_module(module_id)
10848            .map_err(SuperviseError::Registry)?;
10849        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10850            Ok(()) => return Ok(()),
10851            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10852            Err(err) => return Err(err),
10853        }
10854        let pid = lock_snapshot(snapshot)?.reaped_pid;
10855        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10856            warn!(
10857                module_id,
10858                pid,
10859                connection_id = registration.connection_id.get(),
10860                "reaped module registration outlived release grace; closing dead connection"
10861            );
10862            forwarding.request_connection_close(
10863                registration.connection_id,
10864                CloseReason::new(
10865                    "supervised_process_reaped",
10866                    format!("module '{module_id}' pid {pid} exited"),
10867                ),
10868            );
10869            wait_for_slot_registration_release(
10870                registry,
10871                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10872                REGISTRY_RELEASE_TIMEOUT,
10873            )
10874            .await?;
10875        }
10876        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10877    }
10878    .await;
10879    if let Err(err) = &result {
10880        fail_snapshot(snapshot, Some(module_id), None);
10881        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10882    }
10883    result
10884}
10885
10886/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10887/// plain stop or restart waits for before it spawns a replacement.
10888async fn wait_for_registration_release(
10889    registry: &Registry,
10890    module_id: &str,
10891    wait: Duration,
10892) -> Result<(), SuperviseError> {
10893    wait_for_slot_registration_release(
10894        registry,
10895        crate::registry::RegistrationSlot::Active(module_id),
10896        wait,
10897    )
10898    .await
10899}
10900
10901/// Wait for the registration in `slot` to go away.
10902///
10903/// Keyed on the slot rather than the bare module id because a successful swap
10904/// never empties the id's active slot (the promoted candidate is in it), so an
10905/// id-keyed wait for the incumbent's release would always time out. Draining a
10906/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10907/// incumbent's connection instead.
10908async fn wait_for_slot_registration_release(
10909    registry: &Registry,
10910    slot: crate::registry::RegistrationSlot<'_>,
10911    wait: Duration,
10912) -> Result<(), SuperviseError> {
10913    let deadline = Instant::now() + wait;
10914    let mut release_events = registration_release_events().subscribe();
10915    let still_active = |registration: &crate::registry::ModuleRegistration| {
10916        SuperviseError::RegistrationStillActive {
10917            module_id: registration.manifest.module_id.clone(),
10918            waited: wait,
10919        }
10920    };
10921    loop {
10922        let _observed_generation = *release_events.borrow_and_update();
10923        let Some(registration) = registry
10924            .registration(slot)
10925            .map_err(SuperviseError::Registry)?
10926        else {
10927            return Ok(());
10928        };
10929
10930        let now = Instant::now();
10931        if now >= deadline {
10932            return Err(still_active(&registration));
10933        }
10934
10935        let remaining = deadline.saturating_duration_since(now);
10936        match timeout(remaining, release_events.changed()).await {
10937            Ok(Ok(())) | Ok(Err(_)) => {}
10938            Err(_) => return Err(still_active(&registration)),
10939        }
10940    }
10941}
10942
10943#[cfg(test)]
10944mod slot_registration_wait_tests {
10945    use super::*;
10946    use crate::registry::{ConnectionId, RegistrationSlot};
10947    use subc_protocol::manifest::ModuleManifest;
10948
10949    #[tokio::test]
10950    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10951        let registry = Arc::new(Registry::default());
10952        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10953        let runtime = supervisor.runtime_config();
10954        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10955        let spec = ModuleSpec {
10956            module_id: "enable-stale-registration".to_string(),
10957            program: PathBuf::from("/missing/enable-retry-test"),
10958            args: Vec::new(),
10959            env: Vec::new(),
10960            reserved: false,
10961            reserved_prefixes: Vec::new(),
10962            protocol: ModuleProtocol::Subc,
10963            overlap: Default::default(),
10964        };
10965        let connection = ConnectionId::new(90);
10966        registry
10967            .register_with_control_ops(
10968                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10969                1,
10970                connection,
10971                Vec::new(),
10972            )
10973            .unwrap();
10974        let mut child = None;
10975        let err = set_child_enabled(
10976            &spec,
10977            &runtime,
10978            &registry,
10979            &supervisor.process_liveness,
10980            &snapshot,
10981            &mut child,
10982            true,
10983        )
10984        .await
10985        .unwrap_err();
10986        assert!(matches!(
10987            err,
10988            SuperviseError::RegistrationStillActive { .. }
10989        ));
10990        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10991        assert!(child.is_none());
10992        registry.deregister_connection(connection).unwrap();
10993        let err = set_child_enabled(
10994            &spec,
10995            &runtime,
10996            &registry,
10997            &supervisor.process_liveness,
10998            &snapshot,
10999            &mut child,
11000            true,
11001        )
11002        .await
11003        .unwrap_err();
11004        assert!(
11005            matches!(err, SuperviseError::Spawn { .. }),
11006            "second enable must attempt a spawn: {err}"
11007        );
11008        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11009    }
11010
11011    const INCUMBENT: u64 = 1;
11012    const CANDIDATE: u64 = 2;
11013
11014    fn swapped_registry() -> Arc<Registry> {
11015        let registry = Arc::new(Registry::default());
11016        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11017        registry
11018            .register_with_control_ops(
11019                manifest.clone(),
11020                1,
11021                ConnectionId::new(INCUMBENT),
11022                Vec::new(),
11023            )
11024            .unwrap();
11025        registry
11026            .register_candidate_with_control_ops(
11027                manifest,
11028                1,
11029                ConnectionId::new(CANDIDATE),
11030                Vec::new(),
11031            )
11032            .unwrap();
11033        registry
11034    }
11035
11036    /// After a promotion the id's active slot is held by the new process, so an
11037    /// id-keyed wait for the incumbent's release can never succeed; the
11038    /// connection-keyed wait completes as soon as the incumbent deregisters.
11039    #[tokio::test]
11040    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11041        let registry = swapped_registry();
11042        registry.promote_candidate("m").unwrap().unwrap();
11043
11044        assert!(matches!(
11045            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11046            Err(SuperviseError::RegistrationStillActive { .. })
11047        ));
11048
11049        // Still held while the incumbent's connection has not deregistered.
11050        assert!(matches!(
11051            wait_for_slot_registration_release(
11052                &registry,
11053                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11054                Duration::from_millis(50),
11055            )
11056            .await,
11057            Err(SuperviseError::RegistrationStillActive { .. })
11058        ));
11059
11060        let releaser = Arc::clone(&registry);
11061        let release = tokio::spawn(async move {
11062            sleep(Duration::from_millis(20)).await;
11063            releaser
11064                .deregister_connection(ConnectionId::new(INCUMBENT))
11065                .unwrap();
11066            notify_registration_release();
11067        });
11068        wait_for_slot_registration_release(
11069            &registry,
11070            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11071            Duration::from_secs(5),
11072        )
11073        .await
11074        .expect("the incumbent's own registration is released");
11075        release.await.unwrap();
11076        assert!(registry.get_module("m").unwrap().is_some());
11077    }
11078
11079    /// The candidate slot is waited on separately from the active slot: the
11080    /// incumbent's registration neither holds up nor stands in for it.
11081    #[tokio::test]
11082    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11083        let registry = swapped_registry();
11084        assert!(matches!(
11085            wait_for_slot_registration_release(
11086                &registry,
11087                RegistrationSlot::Candidate("m"),
11088                Duration::from_millis(50),
11089            )
11090            .await,
11091            Err(SuperviseError::RegistrationStillActive { .. })
11092        ));
11093        registry
11094            .deregister_connection(ConnectionId::new(CANDIDATE))
11095            .unwrap();
11096        wait_for_slot_registration_release(
11097            &registry,
11098            RegistrationSlot::Candidate("m"),
11099            Duration::from_millis(50),
11100        )
11101        .await
11102        .expect("a candidate slot with no candidate is released");
11103        assert!(registry
11104            .registration(RegistrationSlot::Active("m"))
11105            .unwrap()
11106            .is_some());
11107    }
11108}
11109
11110fn classify_exit(status: &ExitStatus) -> ExitReport {
11111    ExitReport {
11112        kind: if status.success() {
11113            ExitKind::Clean
11114        } else {
11115            ExitKind::Crash
11116        },
11117        code: status.code(),
11118        signal: exit_signal(status),
11119        at_ms: unix_ms_now(),
11120    }
11121}
11122
11123/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11124/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11125/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11126/// disposition still must be `Failed` so the terminal ring is not silently missing
11127/// an entry, matching what `fail_snapshot` records for this same arm.
11128fn wait_error_exit_report() -> ExitReport {
11129    ExitReport {
11130        kind: ExitKind::Crash,
11131        code: None,
11132        signal: None,
11133        at_ms: unix_ms_now(),
11134    }
11135}
11136
11137#[cfg(unix)]
11138fn exit_signal(status: &ExitStatus) -> Option<i32> {
11139    use std::os::unix::process::ExitStatusExt;
11140
11141    status.signal()
11142}
11143
11144#[cfg(not(unix))]
11145fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11146    None
11147}
11148
11149/// Give an operator-touched module its full crash budget back.
11150///
11151/// Named for the counter it used to zero; it now empties the in-window ring,
11152/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11153/// ledger of what happened survives every operator action.
11154fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11155    update_snapshot(snapshot, Some(module_id), |state| {
11156        state.clear_crash_restarts();
11157    })
11158}
11159
11160fn set_running(
11161    snapshot: &SharedSnapshot,
11162    child: &SupervisedChild,
11163    module_id: &str,
11164    spawn_events: &SpawnEventFeed,
11165) -> Result<(), SuperviseError> {
11166    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11167        module_id: Some(module_id.to_string()),
11168    })?;
11169    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11170    if std::mem::take(&mut state.coalesced_restart_pending) {
11171        let generation = state.spawn_generation;
11172        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11173    }
11174    state.drain_disposition_detail = None;
11175    state.spawn_failure = None;
11176    state.waiting_for_previous_pid = None;
11177    state.waiting_for_previous_detail = None;
11178    // Every caller of this is a plain spawn, which always uses the primary key;
11179    // a promoted swap candidate sets the flag itself after this returns.
11180    state.in_alternate_slot = false;
11181    state.configuration_updated_since_spawn = false;
11182    state.spawned_protocol = Some(child.protocol);
11183    state.state = ModuleState::Running;
11184    state.enabled = true;
11185    state.process_alive = true;
11186    state.pid = child.id();
11187    #[cfg(windows)]
11188    {
11189        state.pipe_consumed = child
11190            .nonce_delivery
11191            .as_ref()
11192            .map(|delivery| delivery.consumed());
11193    }
11194    #[cfg(target_os = "macos")]
11195    {
11196        state.report_ready = Some(Arc::clone(&child.report_ready));
11197    }
11198    state.spawned_at_ms = Some(child.spawned_at_ms);
11199    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11200    state.spawned_from = Some(child.spawned_from.clone());
11201    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11202    state.spawned_file_identity = child.spawned_file_identity.clone();
11203    state.process_start_time = child.process_start_time;
11204    Ok(())
11205}
11206
11207fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11208    state.waiting_for_previous_pid = None;
11209    state.waiting_for_previous_detail = None;
11210    state.process_alive = false;
11211    state.spawned_protocol = None;
11212    state.pid = None;
11213    #[cfg(windows)]
11214    {
11215        state.pipe_consumed = None;
11216    }
11217    #[cfg(target_os = "macos")]
11218    {
11219        state.report_ready = None;
11220    }
11221    state.spawned_at_ms = None;
11222    state.exec_outcome = None;
11223    state.spawned_from = None;
11224    state.launch_spec_digest = None;
11225    state.spawned_file_identity = None;
11226    state.process_start_time = None;
11227    state.deliberate_severance = None;
11228}
11229
11230#[cfg(test)]
11231fn record_deliberate_severance(
11232    snapshot: &SharedSnapshot,
11233    identity: ProcessIdentity,
11234) -> Result<(), SuperviseError> {
11235    update_snapshot(snapshot, None, |state| {
11236        state.deliberate_severance = Some(identity);
11237    })
11238}
11239
11240fn apply_deliberate_severance_marker(
11241    snapshot: &SharedSnapshot,
11242    exited_identity: Option<ProcessIdentity>,
11243    mut exit_report: ExitReport,
11244) -> ExitReport {
11245    let marker = lock_snapshot(snapshot)
11246        .ok()
11247        .and_then(|mut state| state.deliberate_severance.take());
11248    if marker.is_some() && marker == exited_identity {
11249        exit_report.kind = ExitKind::DeliberateSeverance;
11250    }
11251    exit_report
11252}
11253
11254fn classify_reaped_child_exit(
11255    snapshot: &SharedSnapshot,
11256    child: &SupervisedChild,
11257    status: &ExitStatus,
11258) -> ExitReport {
11259    let _ = update_snapshot(snapshot, None, |state| {
11260        state.reaped_pid = Some(child.pid);
11261        state.spawn_failure = child.spawn_failure.clone();
11262    });
11263    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11264}
11265
11266fn fail_snapshot(
11267    snapshot: &SharedSnapshot,
11268    module_id: Option<&str>,
11269    last_exit: Option<ExitReport>,
11270) {
11271    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11272        state.state = ModuleState::Failed;
11273        clear_current_process_facts(state);
11274        if let Some(last_exit) = last_exit {
11275            state.last_exit = Some(last_exit);
11276        }
11277    }) {
11278        error!(error = %err, "failed to mark supervisor state failed");
11279    }
11280}
11281
11282fn update_snapshot(
11283    snapshot: &SharedSnapshot,
11284    module_id: Option<&str>,
11285    update: impl FnOnce(&mut SupervisorSnapshot),
11286) -> Result<(), SuperviseError> {
11287    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11288        module_id: module_id.map(ToOwned::to_owned),
11289    })?;
11290    update(&mut state);
11291    Ok(())
11292}
11293
11294const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11295
11296fn lock_snapshot_for_control<'a>(
11297    snapshot: &'a SharedSnapshot,
11298    module_id: &str,
11299    caller: &'static str,
11300) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11301    let started_at = Instant::now();
11302    let guard = lock_snapshot(snapshot)?;
11303    let waited = started_at.elapsed();
11304    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11305        warn!(
11306            module_id = %module_id,
11307            waited_ms = waited.as_millis() as u64,
11308            caller = %caller,
11309            "slow snapshot lock"
11310        );
11311    }
11312    Ok(guard)
11313}
11314
11315fn lock_snapshot(
11316    snapshot: &SharedSnapshot,
11317) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11318    snapshot
11319        .lock()
11320        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11321}
11322
11323#[cfg(test)]
11324mod terminal_history_tests {
11325    use std::{
11326        path::PathBuf,
11327        sync::Arc,
11328        time::{Duration, Instant},
11329    };
11330
11331    use tokio::{sync::mpsc, time::sleep};
11332
11333    use super::{
11334        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11335        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11336        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11337        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11338        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11339        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11340        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11341        SupervisorHealthStatus, SupervisorSnapshot,
11342    };
11343    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11344    // use for their own wall-clock deadlines: crash-restart instants must be on
11345    // the same clock the production code stamps them with, which is tokio's (and
11346    // is what `start_paused` tests can move).
11347    use super::Instant as ClockInstant;
11348    use crate::{
11349        registry::Registry,
11350        terminal_ring::{TerminalRing, TerminalRingConfig},
11351    };
11352    use std::sync::Mutex;
11353    use subc_control::TerminalDisposition;
11354
11355    /// See the twin in `control.rs` for why this derives the path from
11356    /// `current_exe()` and why the existence check is here: `--lib` alone does
11357    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11358    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11359    pub(super) fn fake_aft_stub_path() -> PathBuf {
11360        let mut path = std::env::current_exe().expect("current_exe available in tests");
11361        path.pop();
11362        path.pop();
11363        path.push(if cfg!(windows) {
11364            "fake-aft-stub.exe"
11365        } else {
11366            "fake-aft-stub"
11367        });
11368        assert!(
11369            path.exists(),
11370            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11371             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11372            path.display()
11373        );
11374        path
11375    }
11376
11377    #[test]
11378    fn reserved_never_spawned_refuses_every_hello() {
11379        // The canary hole: a reserved id whose module has never spawned had NO
11380        // gate entry and admitted anyone -- the reservation protected the nonce
11381        // holder, not the NAME. Now the entry is present with no legitimate
11382        // holder and refuses all comers.
11383        let supervisor = SupervisorHandle::default();
11384        supervisor.apply_identity_configuration(&ModuleSpec {
11385            module_id: "never-spawned".to_string(),
11386            program: PathBuf::from("/usr/bin/false"),
11387            args: Vec::new(),
11388            env: Vec::new(),
11389            reserved: true,
11390            reserved_prefixes: Vec::new(),
11391            protocol: ModuleProtocol::Subc,
11392            overlap: Default::default(),
11393        });
11394        assert!(
11395            supervisor
11396                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11397                .is_some(),
11398            "forged nonce must refuse on a reserved never-spawned id"
11399        );
11400        assert!(
11401            supervisor
11402                .reserved_hello_rejection("never-spawned", None)
11403                .is_some(),
11404            "absent nonce must refuse on a reserved never-spawned id"
11405        );
11406        // And a real spawn nonce minted later admits exactly that nonce.
11407        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11408        supervisor.apply_identity_configuration(&ModuleSpec {
11409            module_id: "never-spawned".to_string(),
11410            program: PathBuf::from("/usr/bin/false"),
11411            args: Vec::new(),
11412            env: Vec::new(),
11413            reserved: true,
11414            reserved_prefixes: Vec::new(),
11415            protocol: ModuleProtocol::Subc,
11416            overlap: Default::default(),
11417        });
11418        assert!(supervisor
11419            .reserved_hello_rejection("never-spawned", Some("minted"))
11420            .is_none());
11421        assert!(supervisor
11422            .reserved_hello_rejection("never-spawned", Some("forged"))
11423            .is_some());
11424    }
11425
11426    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11427    /// happened, which is what "spent budget" looks like to every reader.
11428    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11429        let now = ClockInstant::now();
11430        for _ in 0..count {
11431            state.crash_restarts.push_back(now);
11432        }
11433    }
11434
11435    /// Age the oldest recorded restart out of `window`, standing in for the hours
11436    /// that would otherwise have to pass. Injecting the instant is the point: a
11437    /// test that slept a real window would take ten minutes and still prove less.
11438    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11439        let aged = state
11440            .crash_restarts
11441            .front()
11442            .expect("a crash restart must be recorded before it can be aged")
11443            .checked_sub(window + Duration::from_secs(1))
11444            .expect("the test clock is far enough from its origin to age an instant");
11445        state.crash_restarts[0] = aged;
11446    }
11447
11448    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11449        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11450        seed_crash_restarts(&mut state, count);
11451        state
11452    }
11453
11454    #[test]
11455    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11456        let policy = RestartPolicy::new(3, Duration::ZERO);
11457        let now = ClockInstant::now();
11458        assert!(daemon_will_restart(
11459            &mut snapshot_with_restarts(true, 2),
11460            &policy,
11461            now
11462        ));
11463        assert!(!daemon_will_restart(
11464            &mut snapshot_with_restarts(true, 3),
11465            &policy,
11466            now
11467        ));
11468        assert!(!daemon_will_restart(
11469            &mut snapshot_with_restarts(false, 0),
11470            &policy,
11471            now
11472        ));
11473    }
11474
11475    #[test]
11476    fn crash_restart_backoff_escalates_with_in_window_count() {
11477        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11478            .with_max_backoff(Duration::from_secs(30));
11479        let now = ClockInstant::now();
11480        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11481        let schedules = (0..4)
11482            .map(|_| {
11483                state
11484                    .next_crash_restart(&policy, now)
11485                    .expect("the test policy allows four crash restarts")
11486            })
11487            .collect::<Vec<_>>();
11488
11489        assert_eq!(
11490            schedules
11491                .iter()
11492                .map(|schedule| schedule.restart_in_window)
11493                .collect::<Vec<_>>(),
11494            vec![0, 1, 2, 3]
11495        );
11496        assert_eq!(
11497            schedules
11498                .iter()
11499                .map(|schedule| schedule.delay)
11500                .collect::<Vec<_>>(),
11501            vec![
11502                Duration::from_millis(100),
11503                Duration::from_secs(1),
11504                Duration::from_secs(10),
11505                Duration::from_secs(30),
11506            ]
11507        );
11508    }
11509
11510    #[test]
11511    fn crash_restart_backoff_resets_after_ring_clear() {
11512        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11513        let now = ClockInstant::now();
11514        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11515        assert_eq!(
11516            state.next_crash_restart(&policy, now).unwrap().delay,
11517            Duration::from_millis(100)
11518        );
11519        assert_eq!(
11520            state.next_crash_restart(&policy, now).unwrap().delay,
11521            Duration::from_secs(1)
11522        );
11523
11524        state.clear_crash_restarts();
11525        let schedule = state
11526            .next_crash_restart(&policy, now)
11527            .expect("a cleared ring must allow another restart");
11528        assert_eq!(schedule.restart_in_window, 0);
11529        assert_eq!(schedule.delay, Duration::from_millis(100));
11530    }
11531
11532    #[test]
11533    fn crash_restart_backoff_ignores_aged_restarts() {
11534        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11535        let now = ClockInstant::now();
11536        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11537        state
11538            .next_crash_restart(&policy, now)
11539            .expect("the first restart is allowed");
11540        state
11541            .next_crash_restart(&policy, now)
11542            .expect("the second restart is allowed");
11543        state.crash_restarts[0] = now
11544            .checked_sub(policy.window + Duration::from_secs(1))
11545            .expect("the fake clock can age a restart past the window");
11546
11547        let schedule = state
11548            .next_crash_restart(&policy, now)
11549            .expect("an aged restart must release its slot");
11550        assert_eq!(schedule.restart_in_window, 1);
11551        assert_eq!(schedule.delay, Duration::from_secs(1));
11552        assert_eq!(state.crash_restarts.len(), 2);
11553    }
11554
11555    /// The budget is a rate: the same three spent restarts refuse a respawn
11556    /// while they are recent and allow one once they have aged past the window.
11557    /// Nothing about the module changed in between, which is the whole point.
11558    #[test]
11559    fn a_budget_spent_before_the_window_no_longer_refuses() {
11560        let policy = RestartPolicy::new(3, Duration::ZERO);
11561        let mut state = snapshot_with_restarts(true, 3);
11562        let now = ClockInstant::now();
11563        assert!(!daemon_will_restart(&mut state, &policy, now));
11564
11565        assert!(daemon_will_restart(
11566            &mut state,
11567            &policy,
11568            now + policy.window + Duration::from_secs(1)
11569        ));
11570        assert!(
11571            state.crash_restarts.is_empty(),
11572            "reading the budget must drop the instants that left the window"
11573        );
11574    }
11575
11576    fn module_with_recovery_snapshot(
11577        state: ModuleState,
11578        enabled: bool,
11579        restart_count: u32,
11580    ) -> SupervisedModule {
11581        let registry = Arc::new(Registry::default());
11582        let supervisor =
11583            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11584        let runtime = supervisor.runtime_config();
11585        let spec = ModuleSpec {
11586            module_id: "recovery-snapshot".to_string(),
11587            program: fake_aft_stub_path(),
11588            args: Vec::new(),
11589            env: Vec::new(),
11590            reserved: false,
11591            reserved_prefixes: Vec::new(),
11592            protocol: ModuleProtocol::Subc,
11593            overlap: Default::default(),
11594        };
11595        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11596        seed_crash_restarts(&mut snapshot, restart_count);
11597        // These tests read synthetic snapshots. A real child and monitor would
11598        // race those reads by replacing the requested state during startup.
11599        let (commands, _rx) = mpsc::channel(4);
11600        SupervisedModule {
11601            inner: Arc::new(SupervisedModuleInner {
11602                module_id: spec.module_id.clone(),
11603                registry,
11604                snapshot: Arc::new(Mutex::new(snapshot)),
11605                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11606                    spec,
11607                    health: runtime.health,
11608                })),
11609                stderr_ring: runtime.stderr_ring,
11610                terminal_ring: runtime.terminal_ring,
11611                commands,
11612                monitor: Mutex::new(None),
11613                restart_policy: runtime.restart_policy,
11614                effective_drain_timeout: runtime.effective_drain_timeout,
11615                provenance_probe: supervisor.provenance_probe.clone(),
11616            }),
11617        }
11618    }
11619
11620    #[cfg(target_os = "linux")]
11621    #[tokio::test]
11622    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11623        let supervisor =
11624            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11625                .with_cgroup_placement(None);
11626        let result = supervisor.spawn(ModuleSpec {
11627            module_id: "no-cgroup-placement".to_string(),
11628            program: fake_aft_stub_path(),
11629            args: Vec::new(),
11630            env: Vec::new(),
11631            reserved: false,
11632            reserved_prefixes: Vec::new(),
11633            protocol: ModuleProtocol::Subc,
11634            overlap: Default::default(),
11635        });
11636
11637        assert!(
11638            result.is_ok(),
11639            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11640        );
11641    }
11642
11643    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11644    async fn undecided_snapshot_uses_shared_restart_predicate() {
11645        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11646            .will_recover_after_connection_loss()
11647            .unwrap());
11648        assert!(
11649            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11650                .will_recover_after_connection_loss()
11651                .unwrap()
11652        );
11653    }
11654
11655    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11656    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11657        assert!(
11658            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11659                .will_recover_after_connection_loss()
11660                .unwrap()
11661        );
11662    }
11663
11664    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11665    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11666        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11667            .will_recover_after_connection_loss()
11668            .unwrap());
11669        assert!(
11670            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11671                .will_recover_after_connection_loss()
11672                .unwrap()
11673        );
11674    }
11675
11676    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11677    async fn warming_snapshot_is_limited_to_startup_phases() {
11678        for state in [
11679            ModuleState::Starting,
11680            ModuleState::Running,
11681            ModuleState::Restarting,
11682        ] {
11683            assert!(
11684                module_with_recovery_snapshot(state, true, 0)
11685                    .is_warming()
11686                    .unwrap(),
11687                "{state:?} should be warming"
11688            );
11689        }
11690        for state in [
11691            ModuleState::Unresponsive,
11692            ModuleState::Draining,
11693            ModuleState::Stopped,
11694            ModuleState::Failed,
11695            ModuleState::Disabled,
11696        ] {
11697            assert!(
11698                !module_with_recovery_snapshot(state, true, 0)
11699                    .is_warming()
11700                    .unwrap(),
11701                "{state:?} should not be warming"
11702            );
11703        }
11704    }
11705
11706    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11707    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11708        let registry = Arc::new(Registry::default());
11709        let supervisor =
11710            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11711        let module = supervisor
11712            .spawn(ModuleSpec {
11713                module_id: "terminal-history".to_string(),
11714                program: fake_aft_stub_path(),
11715                args: Vec::new(),
11716                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11717                reserved: false,
11718                reserved_prefixes: Vec::new(),
11719                protocol: ModuleProtocol::Subc,
11720                overlap: Default::default(),
11721            })
11722            .unwrap();
11723
11724        let deadline = Instant::now() + Duration::from_secs(5);
11725        loop {
11726            let history = module.terminal_history();
11727            if history.entries.len() == 2 {
11728                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11729                assert_eq!(history.dropped, 0);
11730                assert_eq!(
11731                    history
11732                        .entries
11733                        .iter()
11734                        .map(|entry| entry.exit_code)
11735                        .collect::<Vec<_>>(),
11736                    vec![Some(23), Some(23)]
11737                );
11738                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11739                return;
11740            }
11741            assert!(
11742                Instant::now() < deadline,
11743                "module did not retain two terminal exits: {history:?}"
11744            );
11745            sleep(Duration::from_millis(10)).await;
11746        }
11747    }
11748
11749    /// A disable issued while a crash respawn is still backing off must preempt
11750    /// that respawn: the operator's stop wins, the disable must not queue behind
11751    /// the backoff, and the module must never come back up afterwards.
11752    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11753    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11754        let backoff = Duration::from_secs(2);
11755        let supervisor = Supervisor::new_for_test(
11756            Arc::new(Registry::default()),
11757            RestartPolicy::new(10, backoff),
11758        );
11759        let module = supervisor
11760            .spawn(ModuleSpec {
11761                module_id: "disable-during-backoff".to_string(),
11762                program: fake_aft_stub_path(),
11763                args: Vec::new(),
11764                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11765                reserved: false,
11766                reserved_prefixes: Vec::new(),
11767                protocol: ModuleProtocol::Subc,
11768                overlap: Default::default(),
11769            })
11770            .unwrap();
11771
11772        // Wait for the first crash to put the module into its backoff window.
11773        let deadline = Instant::now() + Duration::from_secs(5);
11774        loop {
11775            if module.status().unwrap().state == ModuleState::Restarting {
11776                break;
11777            }
11778            assert!(
11779                Instant::now() < deadline,
11780                "module never entered the crash backoff"
11781            );
11782            sleep(Duration::from_millis(10)).await;
11783        }
11784
11785        let started = Instant::now();
11786        module.set_enabled(false).await.unwrap();
11787        let waited = started.elapsed();
11788
11789        assert!(
11790            waited < backoff / 2,
11791            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11792        );
11793        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11794
11795        // Outlast the backoff: the respawn it was counting down to must never run.
11796        sleep(backoff + Duration::from_millis(500)).await;
11797        let status = module.status().unwrap();
11798        assert_eq!(status.state, ModuleState::Disabled);
11799        assert_eq!(
11800            status.spawn_generation, 1,
11801            "module respawned after the operator disabled it"
11802        );
11803    }
11804
11805    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11806    /// the shape of nats-server, the program this rule exists for.
11807    #[cfg(unix)]
11808    fn protocol_none_sigterm_exits_clean_spec(
11809        module_id: &str,
11810        dir: &std::path::Path,
11811    ) -> (ModuleSpec, PathBuf, PathBuf) {
11812        let ready = dir.join("ready");
11813        let marker = dir.join("sigterm");
11814        let spec = ModuleSpec {
11815            module_id: module_id.to_string(),
11816            program: fake_aft_stub_path(),
11817            args: Vec::new(),
11818            env: vec![
11819                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11820                (
11821                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11822                    marker.display().to_string(),
11823                ),
11824                (
11825                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11826                    ready.display().to_string(),
11827                ),
11828            ],
11829            reserved: false,
11830            reserved_prefixes: Vec::new(),
11831            protocol: ModuleProtocol::None,
11832            overlap: Default::default(),
11833        };
11834        (spec, ready, marker)
11835    }
11836
11837    /// Wait for a file the child writes, so a signal is never sent before the
11838    /// child's SIGTERM handler is installed (the default disposition would
11839    /// kill it by signal and the exit would not be clean).
11840    #[cfg(unix)]
11841    async fn wait_for_file(path: &std::path::Path) {
11842        let deadline = Instant::now() + Duration::from_secs(10);
11843        while !path.exists() {
11844            assert!(
11845                Instant::now() < deadline,
11846                "{} never appeared",
11847                path.display()
11848            );
11849            sleep(Duration::from_millis(10)).await;
11850        }
11851    }
11852
11853    /// A protocol-none module that exits 0 because something OUTSIDE the
11854    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11855    /// the crash-path disposition rather than `stopped`.
11856    #[cfg(unix)]
11857    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11858    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11859        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11860        let (spec, ready, marker) =
11861            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11862        let supervisor = Supervisor::new_for_test(
11863            Arc::new(Registry::default()),
11864            RestartPolicy::new(3, Duration::ZERO),
11865        );
11866        let module = supervisor.spawn(spec).unwrap();
11867        wait_for_file(&ready).await;
11868        // The ready file proves the child installed its SIGTERM handler, not
11869        // that the supervisor has processed the privacy trampoline's exec
11870        // acknowledgement. On macOS status withholds the pid until then.
11871        let deadline = Instant::now() + Duration::from_secs(10);
11872        let first_pid = loop {
11873            let status = module.status().unwrap();
11874            if status.state == ModuleState::Running {
11875                if let Some(pid) = status.pid {
11876                    break pid;
11877                }
11878            }
11879            assert!(
11880                Instant::now() < deadline,
11881                "a running module must report its pid after exec confirmation: {status:?}"
11882            );
11883            sleep(Duration::from_millis(10)).await;
11884        };
11885
11886        rustix::process::kill_process(
11887            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11888            rustix::process::Signal::TERM,
11889        )
11890        .unwrap();
11891
11892        let deadline = Instant::now() + Duration::from_secs(10);
11893        let respawned = loop {
11894            let status = module.status().unwrap();
11895            if status.state == ModuleState::Running
11896                && status.pid.is_some_and(|pid| pid != first_pid)
11897            {
11898                break status;
11899            }
11900            assert!(
11901                Instant::now() < deadline,
11902                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11903            );
11904            sleep(Duration::from_millis(10)).await;
11905        };
11906        assert_eq!(respawned.spawn_generation, 2);
11907        assert!(
11908            marker.exists(),
11909            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11910        );
11911
11912        let history = module.terminal_history();
11913        assert_eq!(history.entries.len(), 1, "{history:?}");
11914        let entry = &history.entries[0];
11915        assert_eq!(entry.exit_code, Some(0));
11916        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11917        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11918
11919        module.stop().await.unwrap();
11920    }
11921
11922    /// Repeated unrequested clean exits of a protocol-none module spend the
11923    /// restart budget exactly as crashes do, and the module ends `failed` with
11924    /// the budget named.
11925    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11926    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11927        let supervisor = Supervisor::new_for_test(
11928            Arc::new(Registry::default()),
11929            RestartPolicy::new(1, Duration::ZERO),
11930        );
11931        let module = supervisor
11932            .spawn(ModuleSpec {
11933                module_id: "none-clean-exit-budget".to_string(),
11934                program: fake_aft_stub_path(),
11935                args: Vec::new(),
11936                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11937                reserved: false,
11938                reserved_prefixes: Vec::new(),
11939                protocol: ModuleProtocol::None,
11940                overlap: Default::default(),
11941            })
11942            .unwrap();
11943
11944        // Failed follows the terminal write; this deadline only bounds a hang,
11945        // not an assumed duration for the two launches or their exit recording.
11946        let deadline = Instant::now() + Duration::from_secs(10);
11947        loop {
11948            let status = module.status().unwrap();
11949            if status.state == ModuleState::Failed {
11950                break;
11951            }
11952            assert!(
11953                Instant::now() < deadline,
11954                "module never exhausted its budget: {status:?} {:?}",
11955                module.terminal_history()
11956            );
11957            sleep(Duration::from_millis(10)).await;
11958        }
11959        let history = module.terminal_history();
11960        assert_eq!(
11961            history
11962                .entries
11963                .iter()
11964                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11965                .collect::<Vec<_>>(),
11966            vec![
11967                (Some(0), TerminalDisposition::Restarting),
11968                (Some(0), TerminalDisposition::Failed),
11969            ]
11970        );
11971        let detail = history.entries[1]
11972            .disposition_detail
11973            .as_deref()
11974            .expect("a budget failure names the budget");
11975        assert!(detail.contains("max_restarts=1"), "{detail}");
11976        assert_eq!(module.status().unwrap().spawn_generation, 2);
11977    }
11978
11979    #[test]
11980    fn restart_budget_failure_is_published_after_its_terminal_record() {
11981        use std::io::{Read, Write};
11982        let supervisor = Supervisor::new_for_test(
11983            Arc::new(Registry::default()),
11984            RestartPolicy::new(0, Duration::ZERO),
11985        );
11986        let runtime = supervisor.runtime_config();
11987        let spec = ModuleSpec {
11988            protocol: ModuleProtocol::None,
11989            ..windowed_crash_spec("budget-publication-order")
11990        };
11991        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11992            ModuleState::Running,
11993            true,
11994        )));
11995        lock_snapshot(&snapshot).unwrap().process_alive = true;
11996        let reaped_snapshot = snapshot.clone();
11997        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11998        let history = runtime.terminal_ring.clone();
11999        let reap = std::thread::spawn(move || {
12000            tokio::runtime::Builder::new_current_thread()
12001                .enable_all()
12002                .build()
12003                .unwrap()
12004                .block_on(on_child_exit(
12005                    &spec,
12006                    runtime.restart_policy,
12007                    &supervisor.registry,
12008                    &reaped_snapshot,
12009                    &runtime.terminal_ring,
12010                    &runtime.spawn_events,
12011                    &runtime.child_roster,
12012                    ExitReport {
12013                        kind: ExitKind::Clean,
12014                        code: Some(0),
12015                        signal: None,
12016                        at_ms: 1,
12017                    },
12018                ))
12019        });
12020        let mut reached = [0];
12021        barrier.read_exact(&mut reached).unwrap();
12022        assert_eq!(&reached, b"R");
12023        let (before_state, before_alive) = {
12024            let state = lock_snapshot(&snapshot).unwrap();
12025            (state.state, state.process_alive)
12026        };
12027        let before_history = history.lock().unwrap().snapshot();
12028        barrier.write_all(b"X").unwrap();
12029        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12030        assert!(before_history.entries.is_empty());
12031        assert_ne!(
12032            before_state,
12033            ModuleState::Failed,
12034            "Failed was visible before its terminal record could be written"
12035        );
12036        assert!(before_alive);
12037        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12038        let history = history.lock().unwrap().snapshot();
12039        assert_eq!(history.entries.len(), 1);
12040        assert_eq!(history.entries[0].exit_code, Some(0));
12041        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12042    }
12043
12044    #[test]
12045    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12046        use std::io::{Read, Write};
12047
12048        let supervisor = Supervisor::new_for_test(
12049            Arc::new(Registry::default()),
12050            RestartPolicy::new(3, Duration::ZERO),
12051        );
12052        let mut runtime = supervisor.runtime_config();
12053        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12054        let snapshot = module.inner.snapshot.clone();
12055        lock_snapshot(&snapshot).unwrap().process_alive = true;
12056        runtime.terminal_ring = module.inner.terminal_ring.clone();
12057        let spec = windowed_crash_spec(module.module_id());
12058        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12059        let reaped_snapshot = snapshot.clone();
12060        let reap = std::thread::spawn(move || {
12061            tokio::runtime::Builder::new_current_thread()
12062                .enable_all()
12063                .build()
12064                .unwrap()
12065                .block_on(on_child_exit(
12066                    &spec,
12067                    runtime.restart_policy,
12068                    &supervisor.registry,
12069                    &reaped_snapshot,
12070                    &runtime.terminal_ring,
12071                    &runtime.spawn_events,
12072                    &runtime.child_roster,
12073                    ExitReport {
12074                        kind: ExitKind::Crash,
12075                        code: Some(2),
12076                        signal: None,
12077                        at_ms: 1,
12078                    },
12079                ))
12080        });
12081
12082        // Hold the crash handler between its budget decision and publication.
12083        // Connection cleanup must still see the recovery that this crash chose,
12084        // even when that recovery consumes the last available restart slot.
12085        let mut reached = [0];
12086        barrier.read_exact(&mut reached).unwrap();
12087        assert_eq!(&reached, b"R");
12088        let before_status = module.status().unwrap();
12089        let before_history = module.terminal_history();
12090        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12091        barrier.write_all(b"X").unwrap();
12092        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12093
12094        assert_eq!(before_status.state, ModuleState::Running);
12095        assert!(before_status.process_alive);
12096        assert_eq!(before_status.restart_count, 3);
12097        assert!(before_history.entries.is_empty());
12098        assert!(
12099            before_recovers,
12100            "connection cleanup would report a terminal close for a scheduled crash restart"
12101        );
12102        let restarting = module.status().unwrap();
12103        assert_eq!(restarting.state, ModuleState::Restarting);
12104        assert_eq!(restarting.restart_count, 3);
12105        assert!(module.will_recover_after_connection_loss().unwrap());
12106        let history = module.terminal_history();
12107        assert_eq!(history.entries.len(), 1);
12108        assert_eq!(
12109            history.entries[0].disposition,
12110            TerminalDisposition::Restarting
12111        );
12112
12113        // Once the replacement is running, its next crash must be judged
12114        // against the exhausted budget rather than the previous recovery.
12115        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12116            state.state = ModuleState::Running;
12117            state.respawn_pending = false;
12118        })
12119        .unwrap();
12120        assert!(!module.will_recover_after_connection_loss().unwrap());
12121        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12122            state.enabled = false;
12123            state.respawn_pending = true;
12124        })
12125        .unwrap();
12126        assert!(!module.will_recover_after_connection_loss().unwrap());
12127    }
12128
12129    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12130        use std::net::{TcpListener, TcpStream};
12131        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12132        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12133        let (barrier, _) = listener.accept().unwrap();
12134        for stream in [&writer, &barrier] {
12135            stream
12136                .set_read_timeout(Some(Duration::from_secs(10)))
12137                .unwrap();
12138            stream
12139                .set_write_timeout(Some(Duration::from_secs(10)))
12140                .unwrap();
12141        }
12142        *events.1.lock().unwrap() = Some(writer);
12143        barrier
12144    }
12145
12146    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12147        use std::io::{Read, Write};
12148        let barrier = events.1.lock().unwrap().take();
12149        if let Some(mut barrier) = barrier {
12150            barrier.write_all(b"R").unwrap();
12151            let mut release = [0];
12152            barrier.read_exact(&mut release).unwrap();
12153            assert_eq!(&release, b"X");
12154        }
12155    }
12156
12157    #[test]
12158    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12159        use std::io::{Read, Write};
12160
12161        let supervisor = Supervisor::new_for_test(
12162            Arc::new(Registry::default()),
12163            RestartPolicy::new(0, Duration::ZERO),
12164        );
12165        let mut runtime = supervisor.runtime_config();
12166        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12167        let snapshot = module.inner.snapshot.clone();
12168        runtime.terminal_ring = module.inner.terminal_ring.clone();
12169        let spec = ModuleSpec {
12170            module_id: module.module_id().into(),
12171            program: fake_aft_stub_path(),
12172            protocol: ModuleProtocol::None,
12173            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12174            ..windowed_crash_spec(module.module_id())
12175        };
12176        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12177        let reap = std::thread::spawn(move || {
12178            tokio::runtime::Builder::new_current_thread()
12179                .enable_all()
12180                .build()
12181                .unwrap()
12182                .block_on(async {
12183                    let mut child =
12184                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12185                    health_restart_child(
12186                        &spec,
12187                        &runtime,
12188                        &supervisor.registry,
12189                        &supervisor.process_liveness,
12190                        &snapshot,
12191                        &mut child,
12192                        SupervisorHealthStatus::Failing,
12193                        Some("health probe did not answer"),
12194                        1,
12195                    )
12196                    .await
12197                    .unwrap();
12198                    assert!(child.is_none());
12199                });
12200        });
12201        // The loopback barrier acknowledges entry into terminal recording by
12202        // health_restart_child's drain, then waits for the test's release byte.
12203        // Status and history are observed while that writer is held, not after
12204        // an elapsed-time guess about when the health action has completed.
12205        let mut reached = [0];
12206        barrier.read_exact(&mut reached).unwrap();
12207        assert_eq!(&reached, b"R");
12208        let before_status = module.status().unwrap();
12209        let before_history = module.terminal_history();
12210        barrier.write_all(b"X").unwrap();
12211        reap.join().unwrap();
12212        assert!(before_history.entries.is_empty());
12213        assert_ne!(
12214            before_status.state,
12215            ModuleState::Failed,
12216            "Failed was visible before its terminal record could be written"
12217        );
12218        assert!(
12219            before_status.process_alive,
12220            "process death was visible before its terminal record could be written"
12221        );
12222        let failed = module.status().unwrap();
12223        assert_eq!(failed.state, ModuleState::Failed);
12224        assert!(!failed.process_alive);
12225        let history = module.terminal_history();
12226        assert_eq!(history.entries.len(), 1);
12227        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12228        assert_eq!(
12229            history.entries[0].disposition_detail.as_deref(),
12230            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12231        );
12232    }
12233
12234    #[test]
12235    fn observed_exits_publish_process_death_after_terminal_history() {
12236        use std::io::{Read, Write};
12237        for (kind, enabled, shutdown, wait_error, expected_state) in [
12238            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12239            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12240            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12241            (
12242                ExitKind::DeliberateSeverance,
12243                true,
12244                false,
12245                false,
12246                ModuleState::Restarting,
12247            ),
12248            #[cfg(unix)]
12249            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12250            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12251        ] {
12252            let supervisor = Supervisor::new_for_test(
12253                Arc::new(Registry::default()),
12254                RestartPolicy::new(1, Duration::ZERO),
12255            );
12256            let runtime = supervisor.runtime_config();
12257            #[cfg(unix)]
12258            if shutdown {
12259                runtime.child_roster.close();
12260            }
12261            #[cfg(not(unix))]
12262            let _ = shutdown;
12263            let spec = windowed_crash_spec("exit-publication-order");
12264            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12265                ModuleState::Running,
12266                enabled,
12267            )));
12268            lock_snapshot(&snapshot).unwrap().process_alive = true;
12269            let reaped_snapshot = snapshot.clone();
12270            let history = runtime.terminal_ring.clone();
12271            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12272            let reap = std::thread::spawn(move || {
12273                if wait_error {
12274                    super::fail_after_wait_error(
12275                        &spec.module_id,
12276                        &reaped_snapshot,
12277                        &runtime.terminal_ring,
12278                        &runtime.spawn_events,
12279                        "wait failed".into(),
12280                    );
12281                } else {
12282                    tokio::runtime::Builder::new_current_thread()
12283                        .enable_all()
12284                        .build()
12285                        .unwrap()
12286                        .block_on(on_child_exit(
12287                            &spec,
12288                            runtime.restart_policy,
12289                            &supervisor.registry,
12290                            &reaped_snapshot,
12291                            &runtime.terminal_ring,
12292                            &runtime.spawn_events,
12293                            &runtime.child_roster,
12294                            ExitReport {
12295                                kind,
12296                                code: Some(0),
12297                                signal: None,
12298                                at_ms: 1,
12299                            },
12300                        ));
12301                }
12302            });
12303            let mut reached = [0];
12304            barrier.read_exact(&mut reached).unwrap();
12305            assert_eq!(&reached, b"R");
12306            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12307            let before_history = history.lock().unwrap().snapshot();
12308            barrier.write_all(b"X").unwrap();
12309            reap.join().unwrap();
12310            assert!(
12311                before_alive,
12312                "{kind:?}: death was published before recording"
12313            );
12314            assert!(before_history.entries.is_empty());
12315            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12316            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12317            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12318        }
12319    }
12320
12321    #[tokio::test]
12322    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12323        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12324        let path = dir.join("terminals.jsonl");
12325        std::fs::create_dir(&path).unwrap();
12326        let supervisor = Supervisor::new_for_test(
12327            Arc::new(Registry::default()),
12328            RestartPolicy::new(0, Duration::ZERO),
12329        )
12330        .with_terminal_journal(path, "health-budget-journal-failure".into());
12331        let runtime = supervisor.runtime_config();
12332        let spec = ModuleSpec {
12333            program: fake_aft_stub_path(),
12334            protocol: ModuleProtocol::None,
12335            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12336            ..windowed_crash_spec("health-budget-journal-failure")
12337        };
12338        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12339        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12340        health_restart_child(
12341            &spec,
12342            &runtime,
12343            &supervisor.registry,
12344            &supervisor.process_liveness,
12345            &snapshot,
12346            &mut child,
12347            SupervisorHealthStatus::Failing,
12348            None,
12349            1,
12350        )
12351        .await
12352        .unwrap();
12353        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12354        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12355        let history = runtime
12356            .terminal_ring
12357            .lock()
12358            .unwrap()
12359            .durable_history(&spec.module_id);
12360        assert!(history.journal_write_failures > 0);
12361        assert_eq!(history.entries.len(), 1);
12362        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12363    }
12364
12365    #[test]
12366    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12367        let supervisor = Supervisor::new_for_test(
12368            Arc::new(Registry::default()),
12369            RestartPolicy::new(0, Duration::ZERO),
12370        );
12371        let runtime = supervisor.runtime_config();
12372        let spec = ModuleSpec {
12373            program: fake_aft_stub_path(),
12374            protocol: ModuleProtocol::None,
12375            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12376            ..windowed_crash_spec("health-budget-recording-panic")
12377        };
12378        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12379        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12380            tokio::runtime::Builder::new_current_thread()
12381                .enable_all()
12382                .build()
12383                .unwrap()
12384                .block_on(async {
12385                    let mut child =
12386                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12387                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12388                    health_restart_child(
12389                        &spec,
12390                        &runtime,
12391                        &supervisor.registry,
12392                        &supervisor.process_liveness,
12393                        &snapshot,
12394                        &mut child,
12395                        SupervisorHealthStatus::Failing,
12396                        None,
12397                        1,
12398                    )
12399                    .await
12400                    .unwrap();
12401                });
12402        }));
12403        let panic = result.expect_err("recording must still unwind");
12404        assert_eq!(
12405            panic.downcast_ref::<String>().map(String::as_str),
12406            Some("spawn event sequence exhausted")
12407        );
12408        let state = lock_snapshot(&snapshot).unwrap();
12409        assert_eq!(state.state, ModuleState::Failed);
12410        assert!(!state.process_alive);
12411        assert!(state.last_exit.is_some());
12412    }
12413
12414    #[tokio::test]
12415    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12416        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12417        let path = dir.join("terminals.jsonl");
12418        std::fs::create_dir(&path).unwrap();
12419        let supervisor = Supervisor::new_for_test(
12420            Arc::new(Registry::default()),
12421            RestartPolicy::new(0, Duration::ZERO),
12422        )
12423        .with_terminal_journal(path, "budget-journal-failure".into());
12424        let runtime = supervisor.runtime_config();
12425        let spec = windowed_crash_spec("budget-journal-failure");
12426        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12427        assert!(matches!(
12428            on_child_exit(
12429                &spec,
12430                runtime.restart_policy,
12431                &supervisor.registry,
12432                &snapshot,
12433                &runtime.terminal_ring,
12434                &runtime.spawn_events,
12435                &runtime.child_roster,
12436                crash_exit_report(1),
12437            )
12438            .await,
12439            NextAction::Stop { .. }
12440        ));
12441        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12442        let history = runtime
12443            .terminal_ring
12444            .lock()
12445            .unwrap()
12446            .durable_history(&spec.module_id);
12447        assert!(history.journal_write_failures > 0);
12448        assert_eq!(history.entries.len(), 1);
12449        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12450    }
12451
12452    #[test]
12453    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12454        let supervisor = Supervisor::new_for_test(
12455            Arc::new(Registry::default()),
12456            RestartPolicy::new(0, Duration::ZERO),
12457        );
12458        let runtime = supervisor.runtime_config();
12459        let spec = windowed_crash_spec("budget-recording-panic");
12460        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12461        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12462        // Exhausting the event sequence makes emit_exited panic before the
12463        // terminal write, exercising publication on the recording unwind.
12464        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12465        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12466            tokio::runtime::Builder::new_current_thread()
12467                .enable_all()
12468                .build()
12469                .unwrap()
12470                .block_on(on_child_exit(
12471                    &spec,
12472                    runtime.restart_policy,
12473                    &supervisor.registry,
12474                    &snapshot,
12475                    &runtime.terminal_ring,
12476                    &runtime.spawn_events,
12477                    &runtime.child_roster,
12478                    crash_exit_report(1),
12479                ))
12480        }));
12481        let panic = result.err().expect("recording must still unwind");
12482        assert_eq!(
12483            panic.downcast_ref::<String>().map(String::as_str),
12484            Some("spawn event sequence exhausted")
12485        );
12486        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12487    }
12488
12489    /// A stop the supervisor itself requests still stops a protocol-none
12490    /// module, even though the child answers the SIGTERM with exit 0.
12491    #[cfg(unix)]
12492    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12493    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12494        for disable in [false, true] {
12495            let label = if disable {
12496                "none-requested-disable"
12497            } else {
12498                "none-requested-stop"
12499            };
12500            let dir = cortexkit_test_support::ScratchDir::new(label);
12501            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12502            let supervisor = Supervisor::new_for_test(
12503                Arc::new(Registry::default()),
12504                RestartPolicy::new(3, Duration::ZERO),
12505            );
12506            let module = supervisor.spawn(spec).unwrap();
12507            wait_for_file(&ready).await;
12508
12509            if disable {
12510                module.set_enabled(false).await.unwrap();
12511            } else {
12512                module.stop().await.unwrap();
12513            }
12514            assert!(
12515                marker.exists(),
12516                "{label}: the child must have left through its SIGTERM handler with exit 0"
12517            );
12518
12519            // Long enough for a zero-backoff respawn to have happened if the
12520            // exit had been treated as a crash.
12521            sleep(Duration::from_millis(500)).await;
12522            let status = module.status().unwrap();
12523            let expected = if disable {
12524                ModuleState::Disabled
12525            } else {
12526                ModuleState::Stopped
12527            };
12528            assert_eq!(status.state, expected, "{label}");
12529            assert_eq!(
12530                status.spawn_generation, 1,
12531                "{label}: respawned after a requested stop"
12532            );
12533            let history = module.terminal_history();
12534            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12535            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12536            assert_ne!(
12537                history.entries[0].disposition,
12538                TerminalDisposition::Restarting,
12539                "{label}"
12540            );
12541        }
12542    }
12543
12544    /// A subc-wire module that exits 0 on its own is still a stop: the
12545    /// protocol-none rule must not reach it.
12546    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12547    async fn subc_wire_clean_exit_is_still_a_stop() {
12548        let supervisor = Supervisor::new_for_test(
12549            Arc::new(Registry::default()),
12550            RestartPolicy::new(3, Duration::ZERO),
12551        );
12552        let module = supervisor
12553            .spawn(ModuleSpec {
12554                module_id: "wire-clean-exit".to_string(),
12555                program: fake_aft_stub_path(),
12556                args: Vec::new(),
12557                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12558                reserved: false,
12559                reserved_prefixes: Vec::new(),
12560                protocol: ModuleProtocol::Subc,
12561                overlap: Default::default(),
12562            })
12563            .unwrap();
12564
12565        let deadline = Instant::now() + Duration::from_secs(10);
12566        while module.terminal_history().entries.is_empty() {
12567            assert!(Instant::now() < deadline, "module never exited");
12568            sleep(Duration::from_millis(10)).await;
12569        }
12570        // Long enough for a zero-backoff respawn to have happened.
12571        sleep(Duration::from_millis(500)).await;
12572        let status = module.status().unwrap();
12573        assert_eq!(status.state, ModuleState::Stopped);
12574        assert_eq!(status.spawn_generation, 1);
12575        let history = module.terminal_history();
12576        assert_eq!(history.entries.len(), 1, "{history:?}");
12577        assert_eq!(history.entries[0].exit_code, Some(0));
12578        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12579    }
12580
12581    #[cfg(unix)]
12582    #[tokio::test]
12583    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12584        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12585        let record = dir.join("live-children.json");
12586        let supervisor = Supervisor::new_for_test(
12587            Arc::new(Registry::default()),
12588            RestartPolicy::new(0, Duration::ZERO),
12589        );
12590        let mut runtime = supervisor.runtime_config();
12591        runtime.child_roster.record_to(record.clone());
12592        let gate = Arc::new(super::ReloadExitRecordGate::default());
12593        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12594        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12595        let spec = ModuleSpec {
12596            module_id: "reload-exit-roster".into(),
12597            program: fake_aft_stub_path(),
12598            args: Vec::new(),
12599            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12600            reserved: false,
12601            reserved_prefixes: Vec::new(),
12602            protocol: ModuleProtocol::Subc,
12603            overlap: Default::default(),
12604        };
12605        let mut child = None;
12606        let reload = super::finish_reload_child(
12607            &spec,
12608            &runtime,
12609            &supervisor.registry,
12610            &supervisor.process_liveness,
12611            &snapshot,
12612            &mut child,
12613        );
12614        tokio::pin!(reload);
12615        tokio::select! {
12616            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12617            _ = gate.reached.notified() => {}
12618        }
12619        assert!(runtime
12620            .terminal_ring
12621            .lock()
12622            .unwrap()
12623            .snapshot()
12624            .entries
12625            .is_empty());
12626        assert_eq!(
12627            crate::live_children::read_record(&record).unwrap().len(),
12628            1,
12629            "shutdown must still wait for the reaped child until its terminal record exists"
12630        );
12631        runtime.child_roster.close();
12632        gate.resume.notify_one();
12633        assert!(reload.await.is_err());
12634        assert!(crate::live_children::read_record(&record)
12635            .unwrap()
12636            .is_empty());
12637        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12638        assert_eq!(history.entries.len(), 1);
12639        assert_eq!(
12640            history.entries[0].disposition,
12641            TerminalDisposition::DaemonShutdown
12642        );
12643    }
12644
12645    /// Each restart-producing arm has its own state transition. Keeping their
12646    /// lifetime count assertions adjacent prevents a later new arm from silently
12647    /// spending budget without recording the historical restart.
12648    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12649    async fn every_restart_increment_path_advances_lifetime_count() {
12650        let supervisor = Supervisor::new_for_test(
12651            Arc::new(Registry::default()),
12652            RestartPolicy::new(1, Duration::ZERO),
12653        );
12654        let runtime = supervisor.runtime_config();
12655        let spec = ModuleSpec {
12656            module_id: "lifetime-increment-path".to_string(),
12657            program: PathBuf::from("/unused/lifetime-increment-path"),
12658            args: Vec::new(),
12659            env: Vec::new(),
12660            reserved: false,
12661            reserved_prefixes: Vec::new(),
12662            protocol: ModuleProtocol::Subc,
12663            overlap: Default::default(),
12664        };
12665
12666        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12667        assert!(matches!(
12668            on_child_exit(
12669                &spec,
12670                runtime.restart_policy,
12671                &supervisor.registry,
12672                &crash_snapshot,
12673                &runtime.terminal_ring,
12674                &runtime.spawn_events,
12675                &runtime.child_roster,
12676                ExitReport {
12677                    kind: ExitKind::Crash,
12678                    code: Some(1),
12679                    signal: None,
12680                    at_ms: 1,
12681                },
12682            )
12683            .await,
12684            NextAction::Restart { schedule: _ }
12685        ));
12686        let (crash_restarts, crash_lifetime) = {
12687            let state = lock_snapshot(&crash_snapshot).unwrap();
12688            (state.crash_restarts.len(), state.lifetime_restarts)
12689        };
12690        assert_eq!(crash_restarts, 1);
12691        assert_eq!(crash_lifetime, 1);
12692
12693        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12694        let mut health_child = None;
12695        assert!(matches!(
12696            health_restart_child(
12697                &spec,
12698                &runtime,
12699                &supervisor.registry,
12700                &supervisor.process_liveness,
12701                &health_snapshot,
12702                &mut health_child,
12703                SupervisorHealthStatus::Failing,
12704                None,
12705                2,
12706            )
12707            .await,
12708            Ok(())
12709        ));
12710        assert!(health_child.is_none());
12711        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12712        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12713        let (health_restarts, health_lifetime) = {
12714            let state = lock_snapshot(&health_snapshot).unwrap();
12715            (state.crash_restarts.len(), state.lifetime_restarts)
12716        };
12717        assert_eq!(health_restarts, 1);
12718        assert_eq!(health_lifetime, 1);
12719
12720        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12721        let mut reload_child = None;
12722        assert!(matches!(
12723            handle_reload_spawn_failure(
12724                &spec,
12725                &runtime,
12726                &supervisor.process_liveness,
12727                &reload_snapshot,
12728                &mut reload_child,
12729                "forced reload spawn failure".to_string(),
12730            )
12731            .await,
12732            Err(SuperviseError::ReloadFailed { .. })
12733        ));
12734        let (reload_restarts, reload_lifetime) = {
12735            let state = lock_snapshot(&reload_snapshot).unwrap();
12736            (state.crash_restarts.len(), state.lifetime_restarts)
12737        };
12738        assert_eq!(reload_restarts, 1);
12739        assert_eq!(reload_lifetime, 1);
12740    }
12741
12742    #[tokio::test]
12743    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12744        let supervisor = Supervisor::new_for_test(
12745            Arc::new(Registry::default()),
12746            RestartPolicy::new(3, Duration::ZERO),
12747        );
12748        let runtime = supervisor.runtime_config();
12749        let spec = ModuleSpec {
12750            module_id: "deliberately-severed".to_string(),
12751            program: PathBuf::from("/unused/deliberately-severed"),
12752            args: Vec::new(),
12753            env: Vec::new(),
12754            reserved: false,
12755            reserved_prefixes: Vec::new(),
12756            protocol: ModuleProtocol::Subc,
12757            overlap: Default::default(),
12758        };
12759        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12760        let process = ProcessIdentity {
12761            pid: 41,
12762            start_time: 101,
12763        };
12764        record_deliberate_severance(&snapshot, process).unwrap();
12765        let exit_report = apply_deliberate_severance_marker(
12766            &snapshot,
12767            Some(process),
12768            ExitReport {
12769                kind: ExitKind::Crash,
12770                code: Some(1),
12771                signal: None,
12772                at_ms: 1,
12773            },
12774        );
12775        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12776
12777        assert!(matches!(
12778            on_child_exit(
12779                &spec,
12780                runtime.restart_policy,
12781                &supervisor.registry,
12782                &snapshot,
12783                &runtime.terminal_ring,
12784                &runtime.spawn_events,
12785                &runtime.child_roster,
12786                exit_report,
12787            )
12788            .await,
12789            NextAction::Restart { schedule: _ }
12790        ));
12791        let state = lock_snapshot(&snapshot).unwrap();
12792        assert_eq!(state.lifetime_restarts, 1);
12793        assert_eq!(state.crash_restarts.len(), 0);
12794    }
12795
12796    #[tokio::test]
12797    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12798        let supervisor = Supervisor::new_for_test(
12799            Arc::new(Registry::default()),
12800            RestartPolicy::new(3, Duration::ZERO),
12801        );
12802        let runtime = supervisor.runtime_config();
12803        let spec = ModuleSpec {
12804            module_id: "genuine-crash".to_string(),
12805            program: PathBuf::from("/unused/genuine-crash"),
12806            args: Vec::new(),
12807            env: Vec::new(),
12808            reserved: false,
12809            reserved_prefixes: Vec::new(),
12810            protocol: ModuleProtocol::Subc,
12811            overlap: Default::default(),
12812        };
12813        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12814
12815        assert!(matches!(
12816            on_child_exit(
12817                &spec,
12818                runtime.restart_policy,
12819                &supervisor.registry,
12820                &snapshot,
12821                &runtime.terminal_ring,
12822                &runtime.spawn_events,
12823                &runtime.child_roster,
12824                ExitReport {
12825                    kind: ExitKind::Crash,
12826                    code: Some(1),
12827                    signal: None,
12828                    at_ms: 1,
12829                },
12830            )
12831            .await,
12832            NextAction::Restart { schedule: _ }
12833        ));
12834        let state = lock_snapshot(&snapshot).unwrap();
12835        assert_eq!(state.lifetime_restarts, 1);
12836        assert_eq!(state.crash_restarts.len(), 1);
12837    }
12838
12839    fn crash_exit_report(at_ms: u64) -> ExitReport {
12840        ExitReport {
12841            kind: ExitKind::Crash,
12842            code: Some(1),
12843            signal: None,
12844            at_ms,
12845        }
12846    }
12847
12848    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12849        ModuleSpec {
12850            module_id: module_id.to_string(),
12851            program: PathBuf::from("/unused").join(module_id),
12852            args: Vec::new(),
12853            env: Vec::new(),
12854            reserved: false,
12855            reserved_prefixes: Vec::new(),
12856            protocol: ModuleProtocol::Subc,
12857            overlap: Default::default(),
12858        }
12859    }
12860
12861    /// A real crash loop still stops. Three crashes with nothing aging out spend
12862    /// a budget of two and the third respawn is refused, and both surfaces an
12863    /// operator has -- the log line and the retained terminal record -- name the
12864    /// window rather than only the cap, because `max_restarts=2` alone is what
12865    /// this budget used to mean.
12866    #[tokio::test]
12867    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12868        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12869        let supervisor = Supervisor::new_for_test(
12870            Arc::new(Registry::default()),
12871            RestartPolicy::new(2, Duration::ZERO),
12872        );
12873        let runtime = supervisor.runtime_config();
12874        let spec = windowed_crash_spec("crash-loop-in-window");
12875        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12876
12877        for attempt in 1..=2 {
12878            assert!(
12879                matches!(
12880                    on_child_exit(
12881                        &spec,
12882                        runtime.restart_policy,
12883                        &supervisor.registry,
12884                        &snapshot,
12885                        &runtime.terminal_ring,
12886                        &runtime.spawn_events,
12887                        &runtime.child_roster,
12888                        crash_exit_report(attempt),
12889                    )
12890                    .await,
12891                    NextAction::Restart { schedule: _ }
12892                ),
12893                "crash {attempt} is inside the budget and must respawn"
12894            );
12895        }
12896
12897        assert!(matches!(
12898            on_child_exit(
12899                &spec,
12900                runtime.restart_policy,
12901                &supervisor.registry,
12902                &snapshot,
12903                &runtime.terminal_ring,
12904                &runtime.spawn_events,
12905                &runtime.child_roster,
12906                crash_exit_report(3),
12907            )
12908            .await,
12909            NextAction::Stop { .. }
12910        ));
12911
12912        {
12913            let state = lock_snapshot(&snapshot).unwrap();
12914            assert_eq!(state.state, ModuleState::Failed);
12915            assert_eq!(state.crash_restarts.len(), 2);
12916            assert_eq!(state.lifetime_restarts, 2);
12917        }
12918
12919        let history = runtime
12920            .terminal_ring
12921            .lock()
12922            .expect("terminal ring is not poisoned")
12923            .snapshot();
12924        let last = history
12925            .entries
12926            .last()
12927            .expect("the refused crash is retained");
12928        assert_eq!(last.disposition, TerminalDisposition::Failed);
12929        assert_eq!(
12930            last.disposition_detail.as_deref(),
12931            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12932        );
12933
12934        let captured = crate::router::test_log::captured_logs(&logs);
12935        assert!(
12936            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12937            "the stop must be logged with its window: {captured}"
12938        );
12939    }
12940
12941    /// The rate, stated as a test: three crashes where the first has aged past
12942    /// the window are two crashes as far as the budget is concerned, so the
12943    /// third respawn is allowed and the ring holds only the two recent ones.
12944    ///
12945    /// This is the case a lifetime counter got wrong -- and the case the daemon
12946    /// now hits routinely, since a module exits non-zero every time its
12947    /// connection to the daemon drops.
12948    #[tokio::test]
12949    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12950        let supervisor = Supervisor::new_for_test(
12951            Arc::new(Registry::default()),
12952            RestartPolicy::new(2, Duration::ZERO),
12953        );
12954        let runtime = supervisor.runtime_config();
12955        let spec = windowed_crash_spec("crash-across-windows");
12956        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12957
12958        for attempt in 1..=2 {
12959            assert!(matches!(
12960                on_child_exit(
12961                    &spec,
12962                    runtime.restart_policy,
12963                    &supervisor.registry,
12964                    &snapshot,
12965                    &runtime.terminal_ring,
12966                    &runtime.spawn_events,
12967                    &runtime.child_roster,
12968                    crash_exit_report(attempt),
12969                )
12970                .await,
12971                NextAction::Restart { schedule: _ }
12972            ));
12973        }
12974
12975        // The oldest crash moves out of the window; nothing else about the
12976        // module changes.
12977        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12978            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12979        })
12980        .unwrap();
12981
12982        assert!(
12983            matches!(
12984                on_child_exit(
12985                    &spec,
12986                    runtime.restart_policy,
12987                    &supervisor.registry,
12988                    &snapshot,
12989                    &runtime.terminal_ring,
12990                    &runtime.spawn_events,
12991                    &runtime.child_roster,
12992                    crash_exit_report(3),
12993                )
12994                .await,
12995                NextAction::Restart { schedule: _ }
12996            ),
12997            "a crash older than the window must not hold a budget slot"
12998        );
12999
13000        let state = lock_snapshot(&snapshot).unwrap();
13001        assert_eq!(state.state, ModuleState::Restarting);
13002        assert_eq!(
13003            state.crash_restarts.len(),
13004            2,
13005            "the aged instant is dropped and the new one takes its place"
13006        );
13007        assert_eq!(
13008            state.lifetime_restarts, 3,
13009            "the ledger counts every restart, including the ones the window forgot"
13010        );
13011    }
13012
13013    /// An operator restart hands the budget back whole, and the ledger keeps
13014    /// counting. Those are different questions -- "how close is this module to
13015    /// being stopped" and "how many times has it been replaced" -- and the
13016    /// operator action answers only the first.
13017    #[tokio::test]
13018    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13019        let supervisor = Supervisor::new_for_test(
13020            Arc::new(Registry::default()),
13021            RestartPolicy::new(2, Duration::ZERO),
13022        );
13023        let runtime = supervisor.runtime_config();
13024        let spec = windowed_crash_spec("operator-cleared-budget");
13025        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13026
13027        for attempt in 1..=2 {
13028            assert!(matches!(
13029                on_child_exit(
13030                    &spec,
13031                    runtime.restart_policy,
13032                    &supervisor.registry,
13033                    &snapshot,
13034                    &runtime.terminal_ring,
13035                    &runtime.spawn_events,
13036                    &runtime.child_roster,
13037                    crash_exit_report(attempt),
13038                )
13039                .await,
13040                NextAction::Restart { schedule: _ }
13041            ));
13042        }
13043
13044        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13045        {
13046            let state = lock_snapshot(&snapshot).unwrap();
13047            assert!(
13048                state.crash_restarts.is_empty(),
13049                "an operator restart returns the full budget"
13050            );
13051            assert_eq!(
13052                state.lifetime_restarts, 2,
13053                "clearing the budget must not unmake the crashes"
13054            );
13055        }
13056
13057        assert!(
13058            matches!(
13059                on_child_exit(
13060                    &spec,
13061                    runtime.restart_policy,
13062                    &supervisor.registry,
13063                    &snapshot,
13064                    &runtime.terminal_ring,
13065                    &runtime.spawn_events,
13066                    &runtime.child_roster,
13067                    crash_exit_report(3),
13068                )
13069                .await,
13070                NextAction::Restart { schedule: _ }
13071            ),
13072            "the cleared budget must be spendable again"
13073        );
13074        let state = lock_snapshot(&snapshot).unwrap();
13075        assert_eq!(state.crash_restarts.len(), 1);
13076        assert_eq!(state.lifetime_restarts, 3);
13077    }
13078
13079    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13080    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13081        let severed = ProcessIdentity {
13082            pid: 41,
13083            start_time: 101,
13084        };
13085        let successor = ProcessIdentity {
13086            pid: 41,
13087            start_time: 202,
13088        };
13089        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13090        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13091            state.pid = Some(successor.pid);
13092            state.process_start_time = Some(successor.start_time);
13093        })
13094        .unwrap();
13095        assert!(!module.record_deliberate_severance(severed).unwrap());
13096
13097        let exit_report = apply_deliberate_severance_marker(
13098            &module.inner.snapshot,
13099            Some(successor),
13100            ExitReport {
13101                kind: ExitKind::Crash,
13102                code: Some(1),
13103                signal: None,
13104                at_ms: 1,
13105            },
13106        );
13107
13108        assert_eq!(exit_report.kind, ExitKind::Crash);
13109    }
13110
13111    #[tokio::test]
13112    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13113        let registry = Registry::default();
13114        let supervisor = Supervisor::new_for_test(
13115            Arc::new(Registry::default()),
13116            RestartPolicy::new(3, Duration::ZERO),
13117        );
13118        let runtime = supervisor.runtime_config();
13119        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13120        let spec = ModuleSpec {
13121            module_id: "drain-deliberate-severance".to_string(),
13122            program: fake_aft_stub_path(),
13123            args: Vec::new(),
13124            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13125            reserved: false,
13126            reserved_prefixes: Vec::new(),
13127            protocol: ModuleProtocol::Subc,
13128            overlap: Default::default(),
13129        };
13130        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13131        let process = ProcessIdentity {
13132            pid: 41,
13133            start_time: 101,
13134        };
13135        child.process_identity = Some(process);
13136        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13137            state.pid = Some(process.pid);
13138            state.process_start_time = Some(process.start_time);
13139        })
13140        .unwrap();
13141        record_deliberate_severance(&snapshot, process).unwrap();
13142
13143        drain_child_to_state(
13144            &spec.module_id,
13145            spec.protocol,
13146            // The child exits on its own; no signal may change the exit this
13147            // test classifies.
13148            StopNotice::SentOverConnection,
13149            &registry,
13150            None,
13151            &snapshot,
13152            &runtime.terminal_ring,
13153            &runtime.spawn_events,
13154            child,
13155            Duration::from_secs(1),
13156            ModuleState::Stopped,
13157            Some(false),
13158        )
13159        .await
13160        .unwrap();
13161
13162        let state = lock_snapshot(&snapshot).unwrap();
13163        assert_eq!(
13164            state.last_exit.as_ref().map(|exit| exit.kind),
13165            Some(ExitKind::DeliberateSeverance)
13166        );
13167        assert_eq!(state.lifetime_restarts, 1);
13168        assert_eq!(state.crash_restarts.len(), 0);
13169        drop(state);
13170        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13171        assert_eq!(
13172            history.entries[0].exit_kind,
13173            subc_control::TerminalExitKind::DeliberateSeverance
13174        );
13175    }
13176
13177    #[tokio::test]
13178    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13179        let registry = Registry::default();
13180        let supervisor = Supervisor::new_for_test(
13181            Arc::new(Registry::default()),
13182            RestartPolicy::new(3, Duration::ZERO),
13183        );
13184        let runtime = supervisor.runtime_config();
13185        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13186        let spec = ModuleSpec {
13187            module_id: "ordinary-drain".to_string(),
13188            program: fake_aft_stub_path(),
13189            args: Vec::new(),
13190            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13191            reserved: false,
13192            reserved_prefixes: Vec::new(),
13193            protocol: ModuleProtocol::Subc,
13194            overlap: Default::default(),
13195        };
13196        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13197
13198        drain_child_to_state(
13199            &spec.module_id,
13200            spec.protocol,
13201            // The child exits on its own; no signal may change the exit this
13202            // test classifies.
13203            StopNotice::SentOverConnection,
13204            &registry,
13205            None,
13206            &snapshot,
13207            &runtime.terminal_ring,
13208            &runtime.spawn_events,
13209            child,
13210            Duration::from_secs(1),
13211            ModuleState::Stopped,
13212            Some(false),
13213        )
13214        .await
13215        .unwrap();
13216
13217        let state = lock_snapshot(&snapshot).unwrap();
13218        assert_eq!(
13219            state.last_exit.as_ref().map(|exit| exit.kind),
13220            Some(ExitKind::Crash)
13221        );
13222        assert_eq!(state.lifetime_restarts, 0);
13223        assert_eq!(state.crash_restarts.len(), 0);
13224    }
13225
13226    #[test]
13227    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13228        // The server's generic fatal-routing branch only knows that the
13229        // connection failed; it does not know that the daemon deliberately
13230        // initiated a process-killing severance. Keep this seam explicit so a
13231        // future connection error path cannot silently reintroduce the stale
13232        // exemption that mislabels a later genuine crash.
13233        assert!(!include_str!("server.rs")
13234            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13235    }
13236
13237    /// The `route.closed` `drained` value must be the quiescence wait's own
13238    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13239    /// measurement at all and `false` is the one honest constant. This is the exact
13240    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13241    /// on every return path, including the one that used to return early via `?`
13242    /// with `route.closing` already sent and no `route.closed` ever following.
13243    #[test]
13244    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13245        assert!(drained_after_quiescence_wait(&Ok(true)));
13246        assert!(!drained_after_quiescence_wait(&Ok(false)));
13247        assert!(!drained_after_quiescence_wait(&Err(
13248            SuperviseError::StatePoisoned { module_id: None }
13249        )));
13250    }
13251
13252    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13253    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13254    /// already reaped out-of-band) still leaves a terminal record rather than none
13255    /// at all. Triggering the real `wait()` I/O error from an integration test would
13256    /// need a genuine already-reaped-child race, which is OS-specific and not
13257    /// something this suite attempts elsewhere; this test instead verifies the
13258    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13259    /// the call site itself is verified by inspection to sit in that exact arm.
13260    #[test]
13261    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13262        let ring = Arc::new(Mutex::new(TerminalRing::new(
13263            TerminalRingConfig::default(),
13264            0,
13265        )));
13266        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13267
13268        let snapshot = ring.lock().unwrap().snapshot();
13269        assert_eq!(snapshot.entries.len(), 1);
13270        let entry = &snapshot.entries[0];
13271        assert_eq!(entry.exit_code, None);
13272        assert_eq!(entry.exit_signal, None);
13273        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13274    }
13275
13276    #[test]
13277    fn wait_error_exit_path_preserves_spawn_event_density() {
13278        let feed = super::SpawnEventFeed::default();
13279        feed.configure_incarnation("wait-error-density".to_string());
13280        feed.emit_spawned("wait-error", 41, 1);
13281        let ring = Arc::new(Mutex::new(TerminalRing::new(
13282            TerminalRingConfig::default(),
13283            0,
13284        )));
13285
13286        record_wait_error_terminal("wait-error", &ring, &feed);
13287        feed.emit_spawned("after-wait-error", 42, 2);
13288
13289        let state = feed.0.lock().unwrap();
13290        let sequences = state
13291            .events
13292            .iter()
13293            .map(|event| event.cursor.seq)
13294            .collect::<Vec<_>>();
13295        assert_eq!(sequences, vec![1, 2, 3]);
13296        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13297        assert_eq!(state.events[1].exit_code, None);
13298        assert_eq!(state.events[1].exit_signal, None);
13299    }
13300
13301    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13302    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13303    /// not a clean exit it never actually observed.
13304    #[test]
13305    fn wait_error_exit_report_is_classified_as_a_crash() {
13306        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13307    }
13308}
13309
13310#[cfg(all(test, unix))]
13311mod restart_ordering_tests {
13312    use super::*;
13313    use std::io::Write;
13314    use tokio::io::AsyncReadExt;
13315
13316    #[test]
13317    #[ignore = "child process fixture"]
13318    fn serving_child() {
13319        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13320            return;
13321        };
13322        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13323        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13324        for stream in listener.incoming() {
13325            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13326        }
13327    }
13328
13329    async fn restart_before_publication(requests: usize) {
13330        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13331        let address_file = home.path().join("address");
13332        let spec = ModuleSpec {
13333            module_id: "restart-serving-child".into(),
13334            program: std::env::current_exe().unwrap(),
13335            args: vec![
13336                "--exact".into(),
13337                "supervise::restart_ordering_tests::serving_child".into(),
13338                "--ignored".into(),
13339                "--nocapture".into(),
13340            ],
13341            env: vec![(
13342                "RESTART_TEST_ADDRESS_FILE".into(),
13343                address_file.display().to_string(),
13344            )],
13345            reserved: false,
13346            reserved_prefixes: vec![],
13347            protocol: ModuleProtocol::None,
13348            overlap: Default::default(),
13349        };
13350        let mut supervisor =
13351            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13352        let (spawned_tx, spawned_rx) = oneshot::channel();
13353        let spawned_tx = Mutex::new(Some(spawned_tx));
13354        let (release_tx, release_rx) = std::sync::mpsc::channel();
13355        let release_rx = Mutex::new(release_rx);
13356        // Hold publication, not the child: it must answer a real request before
13357        // the restart enters the queue, even though status still has generation 0.
13358        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13359            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13360            release_rx
13361                .lock()
13362                .unwrap()
13363                .recv_timeout(Duration::from_secs(20))
13364                .unwrap();
13365        })));
13366        let registered = supervisor
13367            .register_configured_with_health(
13368                spec.clone(),
13369                true,
13370                HealthConfig::default(),
13371                None,
13372                RestartPolicy::new(3, Duration::ZERO),
13373            )
13374            .unwrap();
13375        let module = registered.module.clone();
13376        let startup = tokio::task::spawn_blocking(move || {
13377            supervisor.start_registered_configured(registered, None)
13378        });
13379        spawned_rx.await.unwrap();
13380        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13381            let address = loop {
13382                if let Ok(address) = std::fs::read_to_string(&address_file) {
13383                    if !address.is_empty() {
13384                        break address;
13385                    }
13386                }
13387                tokio::time::sleep(Duration::from_millis(1)).await;
13388            };
13389            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13390            let mut reply = String::new();
13391            stream.read_to_string(&mut reply).await.unwrap();
13392            reply.trim().parse::<u32>().unwrap()
13393        })
13394        .await
13395        .expect("child must serve before generation publication");
13396        assert_eq!(
13397            lock_snapshot(&module.inner.snapshot)
13398                .unwrap()
13399                .spawn_generation,
13400            0
13401        );
13402        let mut replies = Vec::new();
13403        for _ in 0..requests {
13404            let mut restart = Box::pin(module.restart(Some(0)));
13405            // Poll the public API through queue admission, leaving only its ACK
13406            // pending. The monitor cannot consume the command until released.
13407            std::future::poll_fn(|cx| {
13408                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13409                std::task::Poll::Ready(())
13410            })
13411            .await;
13412            replies.push(restart);
13413        }
13414        release_tx.send(()).unwrap();
13415        startup.await.unwrap().unwrap();
13416        for reply in replies {
13417            reply.await.unwrap();
13418        }
13419        // Restart schedules its replacement after completing the handler. Wait
13420        // for that separate actor turn before placing the command barrier.
13421        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13422            loop {
13423                if lock_snapshot(&module.inner.snapshot)
13424                    .unwrap()
13425                    .spawn_generation
13426                    > 1
13427                {
13428                    break;
13429                }
13430                tokio::time::sleep(Duration::from_millis(1)).await;
13431            }
13432        })
13433        .await;
13434        // Commands are handled serially. The update-spec reply proves every
13435        // restart handler has finished before we inspect the snapshot; restart's
13436        // own ACK only acknowledges initiation, not replacement completion.
13437        module.update_spec_for_test(spec).await.unwrap();
13438        let (generation, pid) = {
13439            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13440            (state.spawn_generation, state.pid)
13441        };
13442        module.stop().await.unwrap();
13443        replacement.expect("restart must publish a replacement for the already-serving child");
13444        assert_ne!(
13445            pid,
13446            Some(old_pid),
13447            "a serving child predating the restart must be replaced"
13448        );
13449        assert_eq!(generation, 2, "queued restarts must share one replacement");
13450    }
13451
13452    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13453    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13454        restart_before_publication(1).await;
13455    }
13456
13457    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13458    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13459        restart_before_publication(2).await;
13460    }
13461
13462    #[tokio::test]
13463    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13464        let supervisor =
13465            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13466        let registered = supervisor
13467            .register_configured_with_health(
13468                ModuleSpec {
13469                    module_id: "queued-before-replacement".into(),
13470                    program: "/bin/sh".into(),
13471                    args: vec!["-c".into(), "exec sleep 600".into()],
13472                    env: vec![],
13473                    reserved: false,
13474                    reserved_prefixes: vec![],
13475                    protocol: ModuleProtocol::None,
13476                    overlap: Default::default(),
13477                },
13478                true,
13479                HealthConfig::default(),
13480                None,
13481                RestartPolicy::default(),
13482            )
13483            .unwrap();
13484        let RegisteredConfiguredModule {
13485            mut spec,
13486            mut runtime,
13487            module,
13488            mut commands,
13489        } = registered;
13490        let snapshot = &module.inner.snapshot;
13491        let mut restart = Box::pin(module.restart(Some(0)));
13492        std::future::poll_fn(|cx| {
13493            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13494            std::task::Poll::Ready(())
13495        })
13496        .await;
13497        // Drive the actor explicitly so a replacement is definitely spawned
13498        // after receipt, but before this queued command is dequeued.
13499        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13500        let replacement_pid = child.as_ref().unwrap().id();
13501        let command = commands.recv().await.unwrap();
13502        assert!(
13503            handle_supervisor_command(
13504                command,
13505                &mut spec,
13506                &mut runtime,
13507                &supervisor.registry,
13508                &supervisor.process_liveness,
13509                snapshot,
13510                &mut child,
13511                &mut commands,
13512                &mut VecDeque::new(),
13513            )
13514            .await
13515        );
13516        restart.await.unwrap();
13517        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13518        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13519        if let Some(mut child) = child {
13520            child.start_kill().unwrap();
13521            child.wait().await.unwrap();
13522        }
13523        assert_eq!(
13524            kept_pid, replacement_pid,
13525            "a newer replacement must survive"
13526        );
13527        assert!(
13528            !pending,
13529            "a satisfied restart must not schedule another spawn"
13530        );
13531    }
13532}
13533
13534#[cfg(all(test, unix))]
13535mod health_event_tests {
13536    use super::*;
13537    use cortexkit_test_support::ScratchDir;
13538    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13539
13540    struct Actor {
13541        module: SupervisedModule,
13542        registry: Arc<Registry>,
13543        forwarding: Arc<ForwardingTable>,
13544        spec: ModuleSpec,
13545        health: HealthConfig,
13546        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13547        _home: ScratchDir,
13548    }
13549
13550    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13551        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13552        // for an observed condition, rather than letting paused time auto-advance
13553        // deadlines or assuming a fixed number of yields finishes the work.
13554        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13555        let tick = Instant::now();
13556        loop {
13557            if let Some(value) = observe() {
13558                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13559                return value;
13560            }
13561            assert!(
13562                std::time::Instant::now() < deadline,
13563                "timed out waiting for {reason}"
13564            );
13565            tokio::task::yield_now().await;
13566        }
13567    }
13568
13569    impl Actor {
13570        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13571            let home = ScratchDir::new("health-events");
13572            let registry = Arc::new(Registry::default());
13573            let forwarding = Arc::new(ForwardingTable::default());
13574            let health = HealthConfig {
13575                cadence,
13576                deadline: Duration::from_secs(3600),
13577                ..HealthConfig::default()
13578            };
13579            let supervisor =
13580                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13581                    .with_forwarding(forwarding.clone())
13582                    .with_health_config(health.clone());
13583            let spec = ModuleSpec {
13584                module_id: "event-health-child".into(),
13585                program: PathBuf::from("/bin/sh"),
13586                // The process stays alive but has no real bus peer. Registrations
13587                // below exercise the same registry writes as accepted HELLOs.
13588                args: vec!["-c".into(), "exec sleep 600".into()],
13589                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13590                    .into_iter()
13591                    .map(|key| (key.into(), home.path().display().to_string()))
13592                    .collect(),
13593                reserved: false,
13594                reserved_prefixes: vec![],
13595                protocol,
13596                overlap: Default::default(),
13597            };
13598            let module = supervisor.spawn(spec.clone()).unwrap();
13599            let (_, rx) = mpsc::channel(8);
13600            let actor = Self {
13601                module,
13602                registry,
13603                forwarding,
13604                spec,
13605                health,
13606                rx,
13607                _home: home,
13608            };
13609            actor
13610                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13611                .await;
13612            actor
13613        }
13614
13615        fn turns(&self) -> u64 {
13616            lock_snapshot(&self.module.inner.snapshot)
13617                .unwrap()
13618                .actor_turns
13619        }
13620
13621        async fn wait_for_select(
13622            &self,
13623            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13624        ) -> ActorSelectCheckpoint {
13625            wait_for("actor select checkpoint", || {
13626                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13627                state.actor_select.filter(|checkpoint| {
13628                    checkpoint.generation == state.spawn_generation
13629                        && state.state == ModuleState::Running
13630                        && state.process_alive
13631                        && state.reported_pid().is_some()
13632                        && expected(checkpoint)
13633                })
13634            })
13635            .await
13636        }
13637
13638        fn hello(&mut self, connection: u64, health: bool) {
13639            let manifest =
13640                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13641                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13642                    .build();
13643            let connection = ConnectionId::new(connection);
13644            let (tx, rx) = mpsc::channel(8);
13645            self.rx = rx;
13646            self.forwarding
13647                .register_module_connection(
13648                    connection,
13649                    self.spec.module_id.clone(),
13650                    subc_protocol::PROTOCOL_VERSION,
13651                    Concurrency::ModuleManaged,
13652                    FrameSink::new(tx),
13653                )
13654                .unwrap();
13655            self.registry
13656                .register_with_control_ops(
13657                    manifest,
13658                    subc_protocol::PROTOCOL_VERSION,
13659                    connection,
13660                    if health {
13661                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13662                    } else {
13663                        vec![]
13664                    },
13665                )
13666                .unwrap();
13667        }
13668
13669        async fn probe(&mut self) -> crate::router::OutboundFrame {
13670            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13671                Ok(frame) => Some(frame),
13672                Err(mpsc::error::TryRecvError::Empty) => None,
13673                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13674            })
13675            .await;
13676            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13677            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13678            frame
13679        }
13680
13681        fn no_probe(&mut self) {
13682            assert!(matches!(
13683                self.rx.try_recv(),
13684                Err(mpsc::error::TryRecvError::Empty)
13685            ));
13686        }
13687
13688        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13689            self.forwarding
13690                .complete_module_control_rpc(
13691                    ConnectionId::new(connection),
13692                    frame.header.corr,
13693                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13694                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13695                        status: HealthStatus::Ok,
13696                        detail: None,
13697                        metrics: None,
13698                    }),
13699                )
13700                .unwrap();
13701        }
13702    }
13703
13704    #[tokio::test(start_paused = true)]
13705    async fn no_health_children_park_for_sixty_seconds() {
13706        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13707        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13708        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13709        unadvertised.hello(1, false);
13710        // Include any one-off registration wake in the budget; receiving a
13711        // no-health HELLO must not turn a parked child into a polling child.
13712        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13713        // a single 60 s jump would only observe one expired timer.
13714        for _ in 0..6000 {
13715            tokio::time::advance(Duration::from_millis(10)).await;
13716            tokio::task::yield_now().await;
13717        }
13718        for actor in [&none, &unregistered, &unadvertised] {
13719            assert!(
13720                actor.turns() <= 5,
13721                "no-health actor woke {} times",
13722                actor.turns()
13723            );
13724            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13725        }
13726    }
13727
13728    #[tokio::test(start_paused = true)]
13729    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13730        // With a zero cadence the first probe is due at once, so this measures
13731        // only how fast the registration notification starts health probing,
13732        // not the normal 30-33 s wait before a first probe (checked below).
13733        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13734        let tick = Instant::now();
13735        actor.hello(1, true);
13736        actor.probe().await;
13737        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13738    }
13739
13740    #[tokio::test(start_paused = true)]
13741    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13742        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13743        let hello_tick = Instant::now();
13744        actor.hello(1, true);
13745        let armed = actor
13746            .wait_for_select(|checkpoint| {
13747                checkpoint.registered_connection == Some(ConnectionId::new(1))
13748                    && checkpoint.next_probe_at.is_some()
13749            })
13750            .await;
13751        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13752        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13753        let first_deadline = hello_tick + delay;
13754        assert_eq!(armed.next_probe_at, Some(first_deadline));
13755        tokio::time::advance(Duration::from_secs(29)).await;
13756        let turns = actor.turns();
13757        actor
13758            .registry
13759            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13760            .unwrap();
13761        let updated = actor
13762            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13763            .await;
13764        assert_eq!(
13765            updated.next_probe_at,
13766            Some(first_deadline),
13767            "catalog update must not reset cadence"
13768        );
13769        actor.no_probe();
13770        tokio::time::advance(first_deadline - Instant::now()).await;
13771        let frame = actor.probe().await;
13772        actor.answer(1, frame);
13773        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13774        let next_deadline = Instant::now() + next;
13775        let rearmed = actor
13776            .wait_for_select(|checkpoint| {
13777                checkpoint
13778                    .next_probe_at
13779                    .is_some_and(|deadline| deadline > first_deadline)
13780            })
13781            .await;
13782        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13783        tokio::time::advance(next - Duration::from_nanos(1)).await;
13784        actor.no_probe();
13785        tokio::time::advance(Duration::from_nanos(1)).await;
13786        actor.probe().await;
13787    }
13788
13789    #[tokio::test(start_paused = true)]
13790    async fn disconnect_disarms_and_reconnect_rearms_health() {
13791        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13792        actor.hello(1, true);
13793        actor
13794            .wait_for_select(|checkpoint| {
13795                checkpoint.registered_connection == Some(ConnectionId::new(1))
13796                    && checkpoint.next_probe_at.is_some()
13797            })
13798            .await;
13799        let turns = actor.turns();
13800        let tick = Instant::now();
13801        actor
13802            .registry
13803            .deregister_connection(ConnectionId::new(1))
13804            .unwrap();
13805        let disarmed = actor
13806            .wait_for_select(|checkpoint| {
13807                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13808            })
13809            .await;
13810        assert_eq!(disarmed.next_probe_at, None);
13811        assert_eq!(disarmed.wake_after, None);
13812        assert_eq!(
13813            Instant::now(),
13814            tick,
13815            "disconnect must disarm in the notification tick"
13816        );
13817        tokio::time::advance(Duration::from_secs(60)).await;
13818        actor.no_probe();
13819        let reconnect_tick = Instant::now();
13820        actor.hello(2, true);
13821        let rearmed = actor
13822            .wait_for_select(|checkpoint| {
13823                checkpoint.registered_connection == Some(ConnectionId::new(2))
13824                    && checkpoint.next_probe_at.is_some()
13825            })
13826            .await;
13827        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13828        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13829        tokio::time::advance(delay).await;
13830        actor.probe().await;
13831    }
13832
13833    #[tokio::test(start_paused = true)]
13834    async fn reregistration_without_health_disarms_the_old_schedule() {
13835        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13836        actor.hello(1, true);
13837        actor
13838            .wait_for_select(|checkpoint| {
13839                checkpoint.registered_connection == Some(ConnectionId::new(1))
13840                    && checkpoint.next_probe_at.is_some()
13841            })
13842            .await;
13843        actor
13844            .registry
13845            .deregister_connection(ConnectionId::new(1))
13846            .unwrap();
13847        actor.hello(2, false);
13848        let disarmed = actor
13849            .wait_for_select(|checkpoint| {
13850                checkpoint.registered_connection == Some(ConnectionId::new(2))
13851            })
13852            .await;
13853        assert_eq!(disarmed.next_probe_at, None);
13854        assert_eq!(disarmed.wake_after, None);
13855        tokio::time::advance(Duration::from_secs(60)).await;
13856        actor.no_probe();
13857    }
13858
13859    #[tokio::test(start_paused = true)]
13860    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13861        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13862        actor.hello(1, true);
13863        let armed = actor
13864            .wait_for_select(|checkpoint| {
13865                checkpoint.registered_connection == Some(ConnectionId::new(1))
13866                    && checkpoint.next_probe_at.is_some()
13867            })
13868            .await;
13869        // Simulate the old process's connection closing before the restart
13870        // tears the process down. The replacement process has started but has
13871        // not sent its HELLO, so it is not registered.
13872        actor
13873            .registry
13874            .deregister_connection(ConnectionId::new(1))
13875            .unwrap();
13876        actor.module.restart(Some(0)).await.unwrap();
13877        let parked = actor
13878            .wait_for_select(|checkpoint| {
13879                checkpoint.generation > armed.generation
13880                    && checkpoint.registered_connection.is_none()
13881                    && checkpoint.next_probe_at.is_none()
13882            })
13883            .await;
13884        assert_eq!(parked.wake_after, None);
13885        actor.rx.close();
13886        // Drain the stop notice the restart queued for the old process's connection,
13887        // so it isn't mistaken for traffic to the replacement.
13888        while actor.rx.try_recv().is_ok() {}
13889        let turns = actor.turns();
13890        tokio::time::advance(Duration::from_secs(60)).await;
13891        let after = actor.turns();
13892        eprintln!(
13893            "restart park turns: before={turns} after={after} delta={}",
13894            after - turns
13895        );
13896        assert_eq!(after, turns, "replacement must park before HELLO");
13897        let hello_tick = Instant::now();
13898        actor.hello(2, true);
13899        let rearmed = actor
13900            .wait_for_select(|checkpoint| {
13901                checkpoint.registered_connection == Some(ConnectionId::new(2))
13902                    && checkpoint.next_probe_at.is_some()
13903            })
13904            .await;
13905        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13906        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13907        tokio::time::advance(delay).await;
13908        actor.probe().await;
13909    }
13910
13911    #[tokio::test(start_paused = true)]
13912    async fn rescan_adds_and_removes_http_health_without_polling() {
13913        use tokio::{
13914            io::{AsyncReadExt, AsyncWriteExt},
13915            net::TcpListener,
13916        };
13917        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13918        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13919        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13920        // One millisecond separates successive probes without a polling delay.
13921        actor.health.cadence = Duration::from_millis(1);
13922        let (started_tx, mut started_rx) = mpsc::channel(8);
13923        let peer = tokio::spawn(async move {
13924            loop {
13925                let (mut socket, _) = listener.accept().await.unwrap();
13926                let mut request = [0; 1024];
13927                assert!(socket.read(&mut request).await.unwrap() > 0);
13928                started_tx.send(()).await.unwrap();
13929                socket
13930                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13931                    .await
13932                    .unwrap();
13933            }
13934        });
13935        let tick = Instant::now();
13936        let turns = actor.turns();
13937        actor
13938            .module
13939            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13940            .await
13941            .unwrap();
13942        let armed = actor
13943            .wait_for_select(|checkpoint| {
13944                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13945            })
13946            .await;
13947        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13948        assert_eq!(Instant::now(), tick);
13949        tokio::time::advance(Duration::from_millis(1)).await;
13950        let mut started = false;
13951        wait_for("successful HTTP probe", || {
13952            started |= started_rx.try_recv().is_ok();
13953            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13954                .then_some(())
13955        })
13956        .await;
13957        assert!(started, "added HTTP check must start at its first deadline");
13958        assert_eq!(
13959            actor.module.status().unwrap().health.status,
13960            SupervisorHealthStatus::Ok
13961        );
13962        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13963        actor.health.http = None;
13964        let turns = actor.turns();
13965        actor
13966            .module
13967            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13968            .await
13969            .unwrap();
13970        let parked = actor
13971            .wait_for_select(|checkpoint| {
13972                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13973            })
13974            .await;
13975        assert_eq!(parked.wake_after, None);
13976        let turns = actor.turns();
13977        tokio::time::advance(Duration::from_secs(60)).await;
13978        assert_eq!(
13979            actor.turns(),
13980            turns,
13981            "removed HTTP check must park the actor"
13982        );
13983        assert!(
13984            started_rx.try_recv().is_err(),
13985            "removed HTTP check must stay disarmed"
13986        );
13987        assert_eq!(
13988            actor.module.status().unwrap().health,
13989            ModuleHealthStatus::default()
13990        );
13991        peer.abort();
13992    }
13993}
13994
13995#[cfg(test)]
13996mod drain_busy_reprobe_tests {
13997    //! The drain's busy-gauge wait, driven against an in-process module
13998    //! connection whose health answers the test scripts. Time is paused, so
13999    //! every elapsed duration below is exact virtual time.
14000    use super::*;
14001    use std::collections::BTreeMap;
14002    use std::sync::Mutex as StdMutex;
14003    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14004
14005    const MODULE_ID: &str = "drain-reprobe-module";
14006    const CONNECTION: u64 = 7;
14007    const GAUGE: &str = "runs_in_flight";
14008    /// The production default for both the health cadence and the drain
14009    /// ceiling, which is the pairing that used to cost a full ceiling.
14010    const CADENCE: Duration = Duration::from_secs(30);
14011    const CEILING: Duration = Duration::from_secs(30);
14012
14013    /// How the scripted module answers each `health.check`.
14014    #[derive(Clone, Copy)]
14015    enum Answer {
14016        /// Report the gauge busy until this instant, quiet from then on, after
14017        /// taking `latency` to answer.
14018        BusyUntil { until: Instant, latency: Duration },
14019        /// Answer at once, with metrics that leave the declared gauge out.
14020        OmitsGauge,
14021        /// Never answer.
14022        Silent,
14023    }
14024
14025    struct Drain {
14026        forwarding: Arc<ForwardingTable>,
14027        runtime: SupervisorRuntimeConfig,
14028        endpoint: crate::ModuleEndpointId,
14029        /// When each probe reached the module.
14030        probes: Arc<StdMutex<Vec<Instant>>>,
14031        responder: tokio::task::JoinHandle<()>,
14032    }
14033
14034    impl Drain {
14035        fn start(answer: Answer) -> Self {
14036            let registry = Arc::new(Registry::default());
14037            let forwarding = Arc::new(ForwardingTable::default());
14038            let health = HealthConfig {
14039                cadence: CADENCE,
14040                ..HealthConfig::default()
14041            };
14042            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14043                .with_forwarding(forwarding.clone())
14044                .with_health_config(health)
14045                .runtime_config();
14046            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14047            let endpoint = forwarding
14048                .register_module_connection(
14049                    ConnectionId::new(CONNECTION),
14050                    MODULE_ID.to_string(),
14051                    subc_protocol::PROTOCOL_VERSION,
14052                    Concurrency::ModuleManaged,
14053                    FrameSink::new(tx),
14054                )
14055                .unwrap();
14056            let probes = Arc::new(StdMutex::new(Vec::new()));
14057            let responder = tokio::spawn({
14058                let forwarding = forwarding.clone();
14059                let probes = probes.clone();
14060                async move {
14061                    while let Some(frame) = rx.recv().await {
14062                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14063                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14064                        probes.lock().unwrap().push(Instant::now());
14065                        let metrics = match answer {
14066                            Answer::BusyUntil { until, latency } => {
14067                                sleep(latency).await;
14068                                let busy = u64::from(Instant::now() < until);
14069                                serde_json::json!({ GAUGE: busy })
14070                            }
14071                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14072                            Answer::Silent => continue,
14073                        };
14074                        // A probe abandoned at its deadline has no RPC left to
14075                        // complete, so a late answer may be refused.
14076                        let _ = forwarding.complete_module_control_rpc(
14077                            ConnectionId::new(CONNECTION),
14078                            frame.header.corr,
14079                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14080                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14081                                status: HealthStatus::Ok,
14082                                detail: None,
14083                                metrics: Some(metrics),
14084                            }),
14085                        );
14086                    }
14087                }
14088            });
14089            Self {
14090                forwarding,
14091                runtime,
14092                endpoint,
14093                probes,
14094                responder,
14095            }
14096        }
14097
14098        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14099        /// how long it took.
14100        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14101            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14102            (drained, elapsed)
14103        }
14104
14105        /// [`Self::run`], also returning what the drain's probes read.
14106        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14107            let started = Instant::now();
14108            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14109            let drained = wait_for_forwarding_quiescence(
14110                &self.forwarding,
14111                MODULE_ID,
14112                &self.runtime,
14113                self.endpoint,
14114                started + ceiling,
14115                &[GAUGE.to_string()],
14116                DrainScope::Active,
14117                &mut trace,
14118            )
14119            .await
14120            .unwrap();
14121            (drained, started.elapsed(), trace)
14122        }
14123
14124        fn probes(&self) -> Vec<Instant> {
14125            self.probes.lock().unwrap().clone()
14126        }
14127    }
14128
14129    impl Drop for Drain {
14130        fn drop(&mut self) {
14131            self.responder.abort();
14132        }
14133    }
14134
14135    #[tokio::test(start_paused = true)]
14136    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14137        let drain = Drain::start(Answer::BusyUntil {
14138            until: Instant::now() + Duration::from_secs(1),
14139            latency: Duration::ZERO,
14140        });
14141        let (drained, elapsed) = drain.run(CEILING).await;
14142        assert!(
14143            drained,
14144            "gauges that read zero during the drain must let it finish drained, \
14145             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14146        );
14147        assert!(
14148            elapsed < Duration::from_secs(3),
14149            "drain took {elapsed:?} after the gauges went quiet at 1s"
14150        );
14151        let probes = drain.probes();
14152        assert!(
14153            probes.len() >= 2,
14154            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14155        );
14156    }
14157
14158    /// The same shape in real time on a multi-threaded runtime: the first
14159    /// probe is answered busy at once, within a millisecond of the drain
14160    /// starting, and later probes read idle. The drain must keep probing
14161    /// after that first instant answer and end soon after the gauge falls.
14162    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14163    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14164        let drain = Drain::start(Answer::BusyUntil {
14165            until: Instant::now() + Duration::from_millis(600),
14166            latency: Duration::ZERO,
14167        });
14168        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14169        assert!(drained, "the gauge fell at 600ms: {trace}");
14170        assert!(
14171            elapsed < Duration::from_secs(3),
14172            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14173        );
14174        let probes = drain.probes();
14175        assert!(
14176            probes.len() >= 2 && trace.probes as usize == probes.len(),
14177            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14178        );
14179    }
14180
14181    #[tokio::test(start_paused = true)]
14182    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14183        let started = Instant::now();
14184        let drain = Drain::start(Answer::BusyUntil {
14185            until: started + Duration::from_secs(3600),
14186            latency: Duration::ZERO,
14187        });
14188        let (drained, elapsed) = drain.run(CEILING).await;
14189        assert!(!drained, "a module busy throughout must not drain");
14190        assert!(
14191            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14192            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14193        );
14194        // Re-probing is paced by the interval, not a busy loop: at most one
14195        // probe per interval across the ceiling, plus the first at time zero.
14196        let probes = drain.probes();
14197        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14198        assert!(
14199            probes.len() as u128 <= max_probes,
14200            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14201            probes.len()
14202        );
14203        for pair in probes.windows(2) {
14204            assert!(
14205                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14206                "consecutive probes {:?} apart",
14207                pair[1] - pair[0]
14208            );
14209        }
14210    }
14211
14212    #[tokio::test(start_paused = true)]
14213    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14214        // Each answer takes longer than the re-probe interval; the next probe
14215        // must wait for the previous answer and then the interval.
14216        let latency = Duration::from_secs(1);
14217        let drain = Drain::start(Answer::BusyUntil {
14218            until: Instant::now() + Duration::from_secs(3600),
14219            latency,
14220        });
14221        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14222        assert!(!drained);
14223        let probes = drain.probes();
14224        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14225        for pair in probes.windows(2) {
14226            assert!(
14227                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14228                "a probe started {:?} after the previous one, before it could finish",
14229                pair[1] - pair[0]
14230            );
14231        }
14232    }
14233
14234    #[tokio::test(start_paused = true)]
14235    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14236        let drain = Drain::start(Answer::Silent);
14237        let ceiling = Duration::from_secs(2);
14238        let (drained, elapsed) = drain.run(ceiling).await;
14239        assert!(!drained, "an unanswered probe counts as busy");
14240        assert!(
14241            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14242            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14243        );
14244        assert_eq!(
14245            drain.probes().len(),
14246            1,
14247            "the unanswered probe holds the only slot until the ceiling"
14248        );
14249    }
14250
14251    /// Every event logged while it is installed, as `message` plus fields.
14252    #[derive(Clone, Default)]
14253    struct LogCapture {
14254        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14255    }
14256
14257    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14258        fn on_event(
14259            &self,
14260            event: &tracing::Event<'_>,
14261            _context: tracing_subscriber::layer::Context<'_, S>,
14262        ) {
14263            struct Fields(BTreeMap<String, String>);
14264            impl tracing::field::Visit for Fields {
14265                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14266                    self.0
14267                        .insert(field.name().to_string(), format!("{value:?}"));
14268                }
14269            }
14270            let mut fields = Fields(BTreeMap::new());
14271            event.record(&mut fields);
14272            self.events.lock().unwrap().push(fields.0);
14273        }
14274    }
14275
14276    impl LogCapture {
14277        fn install(&self) -> tracing::subscriber::DefaultGuard {
14278            use tracing_subscriber::layer::SubscriberExt;
14279            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14280        }
14281
14282        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14283            self.events
14284                .lock()
14285                .unwrap()
14286                .iter()
14287                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14288                .cloned()
14289                .collect()
14290        }
14291    }
14292
14293    /// A module that declares a busy gauge but leaves it out of its health
14294    /// answers holds a drain to its ceiling. The timeout line must say so by
14295    /// name, and the drain must warn about the missing gauge once, not on
14296    /// every probe.
14297    #[tokio::test(start_paused = true)]
14298    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14299        let capture = LogCapture::default();
14300        let _guard = capture.install();
14301        let drain = Drain::start(Answer::OmitsGauge);
14302        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14303        assert!(!drained, "a gauge never reported cannot read quiet");
14304        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14305
14306        let missing = capture.with_message("lacks a declared busy gauge");
14307        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14308        assert_eq!(
14309            missing[0].get("missing_gauges").map(String::as_str),
14310            Some(GAUGE)
14311        );
14312
14313        warn_drain_timed_out(
14314            MODULE_ID,
14315            Duration::from_secs(2),
14316            RouteCloseReason::Restart,
14317            &Default::default(),
14318            &trace,
14319        );
14320        let timeouts = capture.with_message("route drain timed out");
14321        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14322        let gauges = &timeouts[0]["busy_gauges"];
14323        assert!(
14324            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14325            "the timeout line must name the gauge and why it read busy: {gauges}"
14326        );
14327        assert!(
14328            gauges.ends_with("; no value ever read"),
14329            "no value was ever read for the gauge: {gauges}"
14330        );
14331        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14332    }
14333
14334    /// The trace names a busy value and when it was read. For a probe that
14335    /// never got an answer, it names the probe's error kind and says that no
14336    /// value was ever read.
14337    #[tokio::test(start_paused = true)]
14338    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14339        let busy = Drain::start(Answer::BusyUntil {
14340            until: Instant::now() + Duration::from_secs(3600),
14341            latency: Duration::ZERO,
14342        });
14343        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14344        let rendered = trace.to_string();
14345        assert!(
14346            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14347                && rendered.contains("; last value non_zero(1) @"),
14348            "{rendered}"
14349        );
14350
14351        let silent = Drain::start(Answer::Silent);
14352        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14353        // The probe is capped at the 1 s ceiling, and its reading is stamped
14354        // when it gave up.
14355        assert_eq!(
14356            trace.to_string(),
14357            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14358        );
14359        assert_eq!(trace.probes, 1);
14360    }
14361
14362    /// A gauge value that is not a count reads busy and is shown as sent.
14363    #[test]
14364    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14365        let gauges = [GAUGE.to_string()];
14366        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14367        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14368        assert_eq!(
14369            busy_gauge_observation(&verdicts),
14370            BusyGaugeObservation::Busy
14371        );
14372        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14373        assert_eq!(
14374            busy_gauge_observation(&zero),
14375            BusyGaugeObservation::Quiescent
14376        );
14377        let none = gauge_verdicts(None, &gauges);
14378        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14379    }
14380}
14381
14382#[cfg(test)]
14383mod health_evidence_tests {
14384    use super::{HealthProbeError, HealthProbeEvidence};
14385    use std::collections::HashSet;
14386
14387    /// The evidential asymmetry, asserted rather than described.
14388    ///
14389    /// Exactly ONE observation is proof a module cannot serve, and the one that
14390    /// fires under CPU starvation is not it. Before the split, all fifteen
14391    /// construction sites collapsed into a single String, so a timeout carried the
14392    /// same weight as a dead lane -- which is how a healthy module was restarted
14393    /// three times in one day.
14394    #[test]
14395    fn only_a_dead_lane_is_proof_of_death() {
14396        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14397        // Three non-proof classes, each for a different reason: silence is
14398        // consistent with health, a bad answer proves the module ALIVE, and a
14399        // daemon-side fault never reached the module at all.
14400        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14401        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14402        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14403    }
14404
14405    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14406    ///
14407    /// A shared label renders two different observations identically in the line an
14408    /// operator reads after an unexplained restart -- the exact confusion this
14409    /// change removes.
14410    #[test]
14411    fn every_evidence_class_has_a_distinct_label() {
14412        let labels = [
14413            HealthProbeError::lane_dead("").label(),
14414            HealthProbeError::no_answer("").label(),
14415            HealthProbeError::bad_answer("").label(),
14416            HealthProbeError::misconfigured("").label(),
14417        ];
14418        let unique: HashSet<_> = labels.iter().collect();
14419        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14420    }
14421
14422    /// The class is additional information, not a replacement.
14423    ///
14424    /// An operator needs both "this was silence" and the specific text saying how
14425    /// long we waited; a classification that swallowed the message would trade one
14426    /// missing distinction for another.
14427    #[test]
14428    fn classification_preserves_the_original_message() {
14429        let err = HealthProbeError::no_answer("module did not answer within 5s");
14430        assert_eq!(err.to_string(), "module did not answer within 5s");
14431        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14432    }
14433}
14434
14435#[cfg(test)]
14436mod health_tombstone_tests {
14437    use std::{path::PathBuf, sync::Arc, time::Duration};
14438
14439    use subc_protocol::{
14440        manifest::Concurrency,
14441        session::{HealthStatus, ModuleControlResponse},
14442    };
14443    use tokio::sync::mpsc;
14444
14445    use super::{
14446        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14447        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14448    };
14449    use crate::{
14450        control::ControlHandler,
14451        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14452        registry::{ConnectionId, Registry},
14453        router::FrameSink,
14454    };
14455
14456    struct ProbeHarness {
14457        spec: ModuleSpec,
14458        runtime: SupervisorRuntimeConfig,
14459        forwarding: Arc<ForwardingTable>,
14460        module_connection: ConnectionId,
14461        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14462        handler: ControlHandler,
14463        module: super::SupervisedModule,
14464    }
14465
14466    fn probe_harness() -> ProbeHarness {
14467        let registry = Arc::new(Registry::default());
14468        let forwarding = Arc::new(ForwardingTable::default());
14469        let supervisor_handle = super::SupervisorHandle::new();
14470        let health = HealthConfig {
14471            http: None,
14472            cadence: Duration::from_secs(30),
14473            deadline: Duration::from_secs(5),
14474            failure_threshold: 3,
14475            on_degraded: HealthAction::Report,
14476            on_failing: HealthAction::Report,
14477            critical: false,
14478        };
14479        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14480            .with_forwarding(Arc::clone(&forwarding))
14481            .with_handle(supervisor_handle.clone())
14482            .with_health_config(health);
14483        let spec = ModuleSpec {
14484            module_id: "late-health-module".to_string(),
14485            program: PathBuf::from("disabled-module"),
14486            args: Vec::new(),
14487            env: Vec::new(),
14488            reserved: false,
14489            reserved_prefixes: Vec::new(),
14490            protocol: ModuleProtocol::Subc,
14491            overlap: Default::default(),
14492        };
14493        let module = supervisor
14494            .supervise_configured(spec.clone(), false)
14495            .unwrap();
14496        let runtime = supervisor.runtime_config();
14497        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14498            .with_supervisor(supervisor_handle);
14499        let module_connection = ConnectionId::new(700);
14500        let (module_tx, module_rx) = mpsc::channel(8);
14501        forwarding
14502            .register_module_connection(
14503                module_connection,
14504                spec.module_id.clone(),
14505                subc_protocol::PROTOCOL_VERSION,
14506                Concurrency::ModuleManaged,
14507                FrameSink::new(module_tx),
14508            )
14509            .unwrap();
14510
14511        ProbeHarness {
14512            spec,
14513            runtime,
14514            forwarding,
14515            module_connection,
14516            module_rx,
14517            handler,
14518            module,
14519        }
14520    }
14521
14522    async fn finish_after(
14523        harness: &mut ProbeHarness,
14524        stall: Duration,
14525    ) -> ModuleControlRpcCompletion {
14526        assert!(stall > harness.runtime.health.deadline);
14527        let deadline = harness.runtime.health.deadline;
14528        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14529        let answer = async {
14530            let frame = harness.module_rx.recv().await.expect("health.check frame");
14531            tokio::time::advance(deadline).await;
14532            tokio::task::yield_now().await;
14533            tokio::time::advance(stall - deadline).await;
14534            harness
14535                .forwarding
14536                .complete_module_control_rpc(
14537                    harness.module_connection,
14538                    frame.header.corr,
14539                    Some("health.check"),
14540                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14541                        status: HealthStatus::Ok,
14542                        detail: None,
14543                        metrics: None,
14544                    }),
14545                )
14546                .unwrap()
14547        };
14548        let (probe_result, completion) = tokio::join!(probe, answer);
14549        let err = probe_result.expect_err("probe must miss its deadline");
14550        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14551        completion
14552    }
14553
14554    async fn time_out_without_answer(harness: &mut ProbeHarness) {
14555        let deadline = harness.runtime.health.deadline;
14556        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14557        let exhaust_deadline = async {
14558            let _frame = harness.module_rx.recv().await.expect("health.check frame");
14559            tokio::time::advance(deadline).await;
14560            tokio::task::yield_now().await;
14561        };
14562        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14563        let err = probe_result.expect_err("probe must miss its deadline");
14564        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14565    }
14566
14567    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14568        let registry = Arc::clone(&harness.module.inner.registry);
14569        let snapshot = Arc::clone(&harness.module.inner.snapshot);
14570        let process_liveness = super::SupervisorProcessLiveness::default();
14571        let mut child = None;
14572        let cycle = super::run_health_probe_cycle(
14573            &harness.spec,
14574            &harness.runtime,
14575            &registry,
14576            &process_liveness,
14577            &snapshot,
14578            &mut child,
14579        );
14580        let peer = async {
14581            let frame = harness.module_rx.recv().await.expect("health.check frame");
14582            if answer {
14583                harness
14584                    .forwarding
14585                    .complete_module_control_rpc(
14586                        harness.module_connection,
14587                        frame.header.corr,
14588                        Some("health.check"),
14589                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14590                            status: HealthStatus::Ok,
14591                            detail: None,
14592                            metrics: Some(serde_json::json!({"ready": true})),
14593                        }),
14594                    )
14595                    .unwrap();
14596            } else {
14597                tokio::time::advance(harness.runtime.health.deadline).await;
14598                tokio::task::yield_now().await;
14599            }
14600        };
14601        tokio::join!(cycle, peer);
14602    }
14603
14604    #[tokio::test(start_paused = true)]
14605    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14606        let mut harness = probe_harness();
14607        // Drive the probe cycle directly with an in-memory wire peer. Stop the
14608        // disabled module's monitor so only this test owns lifecycle transitions;
14609        // no OS process is launched, and a restart is observed at scheduling.
14610        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14611        monitor.abort();
14612        let _ = monitor.await;
14613        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14614            state.enabled = true;
14615            state.state = super::ModuleState::Running;
14616            state.process_alive = true;
14617        })
14618        .unwrap();
14619
14620        run_probe_cycle(&mut harness, true).await;
14621        assert_eq!(
14622            harness.module.status().unwrap().health.status,
14623            super::SupervisorHealthStatus::Ok
14624        );
14625
14626        for failures in 1..harness.runtime.health.failure_threshold {
14627            run_probe_cycle(&mut harness, false).await;
14628            let status = harness.module.status().unwrap();
14629            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14630            assert_eq!(status.health.consecutive_failures, failures);
14631            assert!(status.health.last_probe_ms.is_some());
14632            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14633            assert!(status.health.metrics.is_none());
14634            assert_eq!(status.state, super::ModuleState::Running);
14635            assert!(status.process_alive);
14636            assert_eq!(status.restart_count, 0);
14637            assert_eq!(status.lifetime_restarts, 0);
14638            assert!(status.health.last_action.is_none());
14639            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14640        }
14641
14642        run_probe_cycle(&mut harness, true).await;
14643        let recovered = harness.module.status().unwrap();
14644        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14645        assert_eq!(recovered.health.consecutive_failures, 0);
14646        assert!(recovered.health.detail.is_none());
14647        assert_eq!(
14648            recovered.health.metrics,
14649            Some(serde_json::json!({"ready": true}))
14650        );
14651        assert_eq!(recovered.lifetime_restarts, 0);
14652
14653        for failures in 1..=harness.runtime.health.failure_threshold {
14654            run_probe_cycle(&mut harness, false).await;
14655            let status = harness.module.status().unwrap();
14656            assert_eq!(status.health.consecutive_failures, failures);
14657            if failures < harness.runtime.health.failure_threshold {
14658                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14659                assert_eq!(status.state, super::ModuleState::Running);
14660                assert_eq!(status.lifetime_restarts, 0);
14661                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14662            } else {
14663                assert_eq!(
14664                    status.health.status,
14665                    super::SupervisorHealthStatus::Unresponsive
14666                );
14667                assert_eq!(status.state, super::ModuleState::Restarting);
14668                assert_eq!(status.restart_count, 1);
14669                assert_eq!(status.lifetime_restarts, 1);
14670                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14671                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14672            }
14673        }
14674    }
14675
14676    #[tokio::test(start_paused = true)]
14677    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14678        let mut harness = probe_harness();
14679
14680        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14681        let first_latency = match &first {
14682            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14683            other => panic!("late answer was not retained: {other:?}"),
14684        };
14685        assert!(harness.handler.observe_module_control_completion(first));
14686
14687        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14688        let second_latency = match &second {
14689            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14690            other => panic!("late answer was not retained: {other:?}"),
14691        };
14692        assert!(harness.handler.observe_module_control_completion(second));
14693
14694        assert_eq!(first_latency, Duration::from_secs(8));
14695        assert_eq!(
14696            second_latency - first_latency,
14697            Duration::from_secs(3),
14698            "latency must grow linearly with the additional stall"
14699        );
14700        let health = harness.module.status().unwrap().health;
14701        assert_eq!(health.late_answer_count, 2);
14702        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14703    }
14704
14705    /// A module that answers every probe late must never march to the kill
14706    /// threshold: the late answer proves it is alive, so it must clear the miss
14707    /// streak the timeout recorded. Without the reset, a CPU-starved module
14708    /// that serves every probe seconds past the deadline accumulates
14709    /// `consecutive_failures` to the threshold and is killed — the exact
14710    /// sequence from the 2026-08-14 aft disable, where the daemon logged
14711    /// "proves the module is alive" five times while counting five misses.
14712    #[tokio::test(start_paused = true)]
14713    async fn late_answer_clears_the_consecutive_failure_streak() {
14714        let mut harness = probe_harness();
14715
14716        // Timeout recorded first: the probe path saw no answer in time.
14717        time_out_without_answer(&mut harness).await;
14718        harness
14719            .module
14720            .record_health_probe_failure_for_test("[no-answer] test miss")
14721            .unwrap();
14722        assert_eq!(
14723            harness.module.status().unwrap().health.consecutive_failures,
14724            1,
14725            "precondition: the miss must be on the streak before the late answer"
14726        );
14727
14728        // The stalled reply then lands: proof of life.
14729        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14730        assert!(matches!(
14731            late,
14732            ModuleControlRpcCompletion::LateHealthAnswer { .. }
14733        ));
14734        assert!(harness.handler.observe_module_control_completion(late));
14735
14736        let health = harness.module.status().unwrap().health;
14737        assert_eq!(
14738            health.consecutive_failures, 0,
14739            "a late answer is an answer: the streak must reset"
14740        );
14741        assert_eq!(health.late_answer_count, 1);
14742    }
14743
14744    #[tokio::test(start_paused = true)]
14745    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14746        let mut harness = probe_harness();
14747
14748        for _ in 0..20 {
14749            time_out_without_answer(&mut harness).await;
14750            assert_eq!(
14751                harness.forwarding.health_probe_tombstone_count().unwrap(),
14752                1
14753            );
14754        }
14755    }
14756}
14757
14758#[cfg(test)]
14759mod child_env_tests {
14760    #[cfg(windows)]
14761    use super::{
14762        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14763    };
14764    #[cfg(windows)]
14765    use std::sync::{Arc, OnceLock};
14766    #[cfg(windows)]
14767    #[test]
14768    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14769        let spec = spec(Vec::new());
14770        let mut command = Command::new("unused");
14771        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14772            .unwrap()
14773            .unwrap();
14774        let name = command
14775            .as_std()
14776            .get_envs()
14777            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14778            .and_then(|(_, value)| value)
14779            .unwrap();
14780        assert_eq!(name, OsStr::new(handoff.name()));
14781        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14782        assert!(command.as_std().get_envs().any(|(key, value)| key
14783            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14784            && value == Some(OsStr::new("env"))));
14785        assert!(command
14786            .as_std()
14787            .get_envs()
14788            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14789    }
14790
14791    #[cfg(windows)]
14792    #[test]
14793    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14794        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14795        snapshot.process_alive = true;
14796        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14797        assert_eq!(
14798            observed_launch_nonce_source(&snapshot).as_deref(),
14799            Some("env")
14800        );
14801        let consumed = Arc::new(OnceLock::new());
14802        snapshot.pipe_consumed = Some(consumed.clone());
14803        assert_eq!(
14804            observed_launch_nonce_source(&snapshot).as_deref(),
14805            Some("env")
14806        );
14807        consumed.set(()).unwrap();
14808        assert_eq!(
14809            observed_launch_nonce_source(&snapshot).as_deref(),
14810            Some("pipe")
14811        );
14812        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14813        assert_eq!(
14814            observed_launch_nonce_source(&snapshot).as_deref(),
14815            Some("env")
14816        );
14817        clear_current_process_facts(&mut snapshot);
14818        assert_eq!(observed_launch_nonce_source(&snapshot), None);
14819    }
14820    use super::{
14821        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14822        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14823        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14824    };
14825    use std::{ffi::OsStr, path::PathBuf};
14826    use tokio::process::Command;
14827
14828    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14829        ModuleSpec {
14830            module_id: "env-plan".to_string(),
14831            program: PathBuf::from("/nonexistent"),
14832            args: Vec::new(),
14833            env,
14834            reserved: false,
14835            reserved_prefixes: Vec::new(),
14836            protocol: ModuleProtocol::Subc,
14837            overlap: Default::default(),
14838        }
14839    }
14840
14841    #[test]
14842    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14843        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14844        let digest = LaunchSpecDigest::from_spec(&base);
14845        let mut reordered = base.clone();
14846        reordered.env.reverse();
14847        reordered.module_id = "different-id".into();
14848        reordered.reserved = true;
14849        reordered.overlap = Default::default();
14850        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14851        let mut delimiter_collision = base.clone();
14852        delimiter_collision.env[0] = ("A=B".into(), "C".into());
14853        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14854        let mut changed = base.clone();
14855        changed.protocol = ModuleProtocol::None;
14856        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14857        changed = base.clone();
14858        changed.args.push("--flag".into());
14859        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14860        changed = base.clone();
14861        changed.program = PathBuf::from("/other");
14862        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14863        changed = base.clone();
14864        changed.env[1].1 = "other".into();
14865        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14866        assert!(!format!("{digest:?}").contains("B=C"));
14867        assert!(!format!("{digest:?}").contains("[u8"));
14868    }
14869
14870    #[cfg(unix)]
14871    #[test]
14872    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14873        use std::os::unix::ffi::OsStringExt;
14874        let mut left = spec(Vec::new());
14875        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14876        let mut right = left.clone();
14877        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14878        assert_eq!(
14879            left.program.to_string_lossy(),
14880            right.program.to_string_lossy()
14881        );
14882        assert_ne!(
14883            LaunchSpecDigest::from_spec(&left),
14884            LaunchSpecDigest::from_spec(&right)
14885        );
14886    }
14887
14888    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
14889    /// one still gets its own.
14890    ///
14891    /// This is the narrow goal `env_clear()` was reached for, and the reason the
14892    /// fix is `env_remove` rather than deleting the line: an operator's ambient
14893    /// filter silently becoming an unconfigured module's log level is a real
14894    /// defect, just a much smaller one than clearing the environment.
14895    ///
14896    /// Asserted on the command plan rather than a spawned child because proving
14897    /// the ABSENCE of an inherited variable needs the parent's environment
14898    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
14899    /// removal as `(key, None)`, which is exactly the distinction wanted: not
14900    /// "absent because nobody set it" but "explicitly unset for the child".
14901    #[test]
14902    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14903        let mut command = Command::new("/nonexistent");
14904        apply_child_env(&mut command, &spec(Vec::new()));
14905        let removed = command
14906            .as_std()
14907            .get_envs()
14908            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14909        assert!(
14910            removed,
14911            "ambient CK_LOG must be explicitly removed for an unconfigured module"
14912        );
14913
14914        let mut configured = Command::new("/nonexistent");
14915        apply_child_env(
14916            &mut configured,
14917            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14918        );
14919        let effective = configured
14920            .as_std()
14921            .get_envs()
14922            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14923            .last()
14924            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14925        assert_eq!(
14926            effective,
14927            Some(Some("debug".to_string())),
14928            "a module's configured CK_LOG must survive the ambient removal"
14929        );
14930    }
14931
14932    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
14933    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
14934    /// the same reason as the CK_LOG test above.
14935    ///
14936    /// The argument is the load-bearing half: a stock binary exits on an
14937    /// unknown flag before it listens, so with `--subc` appended the mode
14938    /// could not supervise the one process it exists for. Found by the first
14939    /// conformance run (nats-server: `flag provided but not defined: -subc`).
14940    #[test]
14941    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14942        let connection_file = std::path::Path::new("/run/subc-connection.json");
14943        let handle = SupervisorHandle::new();
14944
14945        let mut none_spec = spec(Vec::new());
14946        none_spec.protocol = ModuleProtocol::None;
14947        let mut none = Command::new("/nonexistent");
14948        let none_handoff =
14949            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14950                .expect("protocol-none spawn args apply");
14951        assert!(
14952            none_handoff.is_none(),
14953            "protocol:none spawn must not receive a nonce descriptor"
14954        );
14955        #[cfg(windows)]
14956        assert!(!none.as_std().get_envs().any(|(key, value)| key
14957            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14958            && value.is_some()));
14959        assert!(
14960            !none.as_std().get_envs().any(|(key, value)| key
14961                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14962                && value.is_some()),
14963            "protocol:none spawn must not name a nonce descriptor"
14964        );
14965        let none_args: Vec<String> = none
14966            .as_std()
14967            .get_args()
14968            .map(|a| a.to_string_lossy().into_owned())
14969            .collect();
14970        assert!(
14971            !none_args.iter().any(|a| a == SUBC_ARG),
14972            "protocol:none argv must not carry --subc; got {none_args:?}"
14973        );
14974        let none_has_nonce = none
14975            .as_std()
14976            .get_envs()
14977            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14978        assert!(
14979            !none_has_nonce,
14980            "protocol:none spawn must not receive a launch nonce"
14981        );
14982        let none_has_module_id = none
14983            .as_std()
14984            .get_envs()
14985            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14986        assert!(
14987            none_has_module_id,
14988            "SUBC_MODULE_ID is inert and stays on every path"
14989        );
14990        assert!(
14991            handle.spawn_nonce(&none_spec.module_id).is_none(),
14992            "no nonce record for a process that will never present one"
14993        );
14994
14995        // Control: the subc-wire path is unchanged by the branch above.
14996        let wire_spec = spec(Vec::new());
14997        let mut wire = Command::new("/nonexistent");
14998        let wire_handoff =
14999            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15000                .expect("subc-wire spawn args apply");
15001        let wire_fd_env = wire
15002            .as_std()
15003            .get_envs()
15004            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15005            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15006        #[cfg(unix)]
15007        assert_eq!(
15008            wire_fd_env,
15009            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15010            "a subc-wire spawn names the pipe it will receive at descriptor 3"
15011        );
15012        #[cfg(windows)]
15013        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15014        #[cfg(not(any(unix, windows)))]
15015        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15016        let wire_args: Vec<String> = wire
15017            .as_std()
15018            .get_args()
15019            .map(|a| a.to_string_lossy().into_owned())
15020            .collect();
15021        assert_eq!(
15022            wire_args,
15023            vec![
15024                SUBC_ARG.to_string(),
15025                connection_file.to_string_lossy().into_owned()
15026            ],
15027            "a subc-wire spawn still carries --subc <path>"
15028        );
15029        assert_eq!(
15030            wire.as_std()
15031                .get_envs()
15032                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15033            !cfg!(unix),
15034            "only Windows supplies the environment nonce"
15035        );
15036        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15037    }
15038
15039    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
15040    /// spec tries to set it; only a swap candidate carries it.
15041    ///
15042    /// "Set it only on candidates" is not enough, because spawn applies the
15043    /// spec's env verbatim and the daemon's own environment is inherited: either
15044    /// could hand a plain restart the swap role, and a module reading it would
15045    /// warm on its long swap budget while callers wait. Asserted as an explicit
15046    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
15047    /// test above gives.
15048    #[test]
15049    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15050        let role = |command: &Command| {
15051            command
15052                .as_std()
15053                .get_envs()
15054                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15055                .last()
15056                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15057        };
15058        let forged = spec(vec![(
15059            SUBC_SPAWN_ROLE_ENV.to_string(),
15060            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15061        )]);
15062
15063        let mut plain = Command::new("/nonexistent");
15064        apply_child_env(&mut plain, &forged);
15065        apply_spawn_role(&mut plain, SpawnRole::Plain);
15066        assert_eq!(
15067            role(&plain),
15068            Some(None),
15069            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15070        );
15071
15072        let mut candidate = Command::new("/nonexistent");
15073        apply_child_env(&mut candidate, &spec(Vec::new()));
15074        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15075        assert_eq!(
15076            role(&candidate),
15077            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15078        );
15079    }
15080
15081    /// Daemon-private capture retention keys never reach the child.
15082    ///
15083    /// cortexkit-log exposes retention as a Rust struct with no environment
15084    /// names, so these entries are supervisor metadata. Passing them through
15085    /// would invent a public child-process contract by accident.
15086    #[test]
15087    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15088        let mut command = Command::new("/nonexistent");
15089        apply_child_env(
15090            &mut command,
15091            &spec(vec![
15092                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15093                ("KEPT".to_string(), "yes".to_string()),
15094            ]),
15095        );
15096        let keys: Vec<String> = command
15097            .as_std()
15098            .get_envs()
15099            .filter(|(_, value)| value.is_some())
15100            .map(|(key, _)| key.to_string_lossy().into_owned())
15101            .collect();
15102        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15103        assert!(
15104            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15105            "daemon-private capture key leaked to the child: {keys:?}"
15106        );
15107    }
15108}
15109
15110#[cfg(test)]
15111mod jitter_tests {
15112    use super::jittered_health_delay;
15113    use std::{collections::HashSet, time::Duration};
15114
15115    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15116    /// that actually occur rather than invented ones.
15117    ///
15118    /// This is a SAMPLE, not a registry: the property under test is that distinct
15119    /// ids disperse, which holds for any set of distinct strings. Several entries
15120    /// are already historical (modules get renamed), and that costs nothing here --
15121    /// but it means a reader must not mistake this for the live module set, and a
15122    /// rename sweep will match it without there being anything to change.
15123    const FLEET: [&str; 14] = [
15124        "aft",
15125        "alfonso-core",
15126        "magic-context",
15127        "broca",
15128        "thalamus",
15129        "quota",
15130        "engram",
15131        "plexus",
15132        "cerebellum",
15133        "astrocyte",
15134        "synapse",
15135        "subc-mcp",
15136        "cortexkit-credentials",
15137        "subc-federation",
15138    ];
15139
15140    /// Probes must not converge after a fleet-wide restart.
15141    ///
15142    /// This is the property the jitter exists for: every module reconnects at
15143    /// once, and without dispersal all fourteen would then probe on the same
15144    /// tick forever. Nothing failed visibly when this went untested -- a
15145    /// convergent fleet still probes correctly, just in a burst, so the symptom
15146    /// is a periodic load spike that looks like whatever else is running.
15147    #[test]
15148    fn probe_delays_disperse_across_the_fleet() {
15149        let cadence = Duration::from_secs(30);
15150        let delays: HashSet<Duration> = FLEET
15151            .iter()
15152            .map(|id| jittered_health_delay(id, 0, cadence))
15153            .collect();
15154        assert_eq!(
15155            delays.len(),
15156            FLEET.len(),
15157            "every supervised module must land on its own probe offset"
15158        );
15159    }
15160
15161    /// The offset may only ever DELAY a probe, never bring it forward.
15162    ///
15163    /// A delay below the cadence would probe a module more often than
15164    /// configured, which is the opposite of what an operator asked for and
15165    /// would tighten the failure budget without anyone changing it.
15166    #[test]
15167    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15168        let cadence = Duration::from_secs(30);
15169        let span = cadence / 10;
15170        for id in FLEET {
15171            for probe_index in 0..8 {
15172                let delay = jittered_health_delay(id, probe_index, cadence);
15173                assert!(
15174                    delay >= cadence,
15175                    "{id}#{probe_index}: jitter must not shorten the cadence"
15176                );
15177                assert!(
15178                    delay < cadence + span,
15179                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15180                );
15181            }
15182        }
15183    }
15184
15185    /// A module keeps its offset across daemon restarts.
15186    ///
15187    /// The delay is derived rather than randomised precisely so a restart does
15188    /// not re-roll every module into a fresh chance of collision. A random
15189    /// source would satisfy the dispersal test above and quietly lose this.
15190    #[test]
15191    fn a_module_offset_is_stable_across_restarts() {
15192        let cadence = Duration::from_secs(30);
15193        for id in FLEET {
15194            assert_eq!(
15195                jittered_health_delay(id, 0, cadence),
15196                jittered_health_delay(id, 0, cadence),
15197                "{id}: the same module and probe index must produce the same offset"
15198            );
15199        }
15200    }
15201
15202    /// A zero cadence disables probing rather than producing a busy loop.
15203    #[test]
15204    fn zero_cadence_yields_zero_delay() {
15205        assert_eq!(
15206            jittered_health_delay("aft", 0, Duration::ZERO),
15207            Duration::ZERO
15208        );
15209    }
15210}
15211
15212#[cfg(all(test, target_os = "linux"))]
15213mod cgroup_placement_tests {
15214    use super::{
15215        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15216        SuperviseError, SupervisedChild,
15217    };
15218    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15219    use cortexkit_test_support::ScratchDir;
15220    use std::{
15221        fs, io,
15222        path::{Path, PathBuf},
15223        sync::{Arc, Mutex},
15224    };
15225    use tokio::process::Command;
15226
15227    #[tokio::test]
15228    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15229        use super::*;
15230        let dir = ScratchDir::new("unique-spawn-cgroups");
15231        let root = PathBuf::from(format!(
15232            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15233            std::process::id(),
15234            unix_ms_now()
15235        ));
15236        if let Err(error) = fs::create_dir(&root) {
15237            assert!(
15238                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15239                "required cgroup test cannot execute: {error}"
15240            );
15241            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15242            return;
15243        }
15244        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15245        let group_count = || {
15246            fs::read_dir(root.join("subc-modules"))
15247                .unwrap()
15248                .map(|entry| entry.unwrap().file_type().unwrap())
15249                .filter(|kind| kind.is_dir())
15250                .count()
15251        };
15252        let supervisor =
15253            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15254                .with_cgroup_placement(Some(placement.clone()));
15255        let runtime = supervisor.runtime_config();
15256        let mut spec = ModuleSpec {
15257            module_id: "unique-spawn".into(),
15258            program: PathBuf::from("/bin/sleep"),
15259            args: vec!["60".into()],
15260            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15261                .into_iter()
15262                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15263                .collect(),
15264            reserved: false,
15265            reserved_prefixes: vec![],
15266            protocol: ModuleProtocol::None,
15267            overlap: Default::default(),
15268        };
15269        let spawn = |spec: &ModuleSpec| {
15270            spawn_child(
15271                spec,
15272                None,
15273                None,
15274                &runtime.stderr_ring,
15275                None,
15276                &runtime.child_roster,
15277                Some(&placement),
15278            )
15279            .unwrap()
15280        };
15281        let mut live = spawn(&spec);
15282        for _ in 0..3 {
15283            // A new process can enter the old slot while retirement is pending.
15284            let next = spawn(&spec);
15285            assert_ne!(live.module_id, next.module_id);
15286            live.start_kill().unwrap();
15287            live.wait().await.unwrap();
15288            live = next;
15289            assert!(
15290                live.child.try_wait().unwrap().is_none(),
15291                "retiring the old slot must not kill the replacement"
15292            );
15293            assert_eq!(
15294                group_count(),
15295                1,
15296                "only the live spawn's cgroup should remain"
15297            );
15298        }
15299        supervisor.begin_daemon_shutdown();
15300        let reap = tokio::spawn(async move {
15301            live.wait().await.unwrap();
15302        });
15303        supervisor
15304            .end_children_for_daemon_shutdown(false, std::future::pending())
15305            .await;
15306        reap.await.unwrap();
15307        assert_eq!(group_count(), 0);
15308        // A normal exit uses the same tree-cleanup path as a killed spawn.
15309        spec.program = PathBuf::from("/bin/true");
15310        spec.args.clear();
15311        let fresh_roster = ChildRoster::default();
15312        let mut short = spawn_child(
15313            &spec,
15314            None,
15315            None,
15316            &runtime.stderr_ring,
15317            None,
15318            &fresh_roster,
15319            Some(&placement),
15320        )
15321        .unwrap();
15322        short.wait().await.unwrap();
15323        assert_eq!(group_count(), 0);
15324        spec.module_id = "_".repeat(255);
15325        let mut long_id = spawn_child(
15326            &spec,
15327            None,
15328            None,
15329            &runtime.stderr_ring,
15330            None,
15331            &fresh_roster,
15332            Some(&placement),
15333        )
15334        .unwrap();
15335        long_id.wait().await.unwrap();
15336        assert_eq!(
15337            group_count(),
15338            0,
15339            "valid long module IDs must not exceed cgroup NAME_MAX"
15340        );
15341        fs::remove_dir(root.join("subc-modules")).unwrap();
15342        fs::remove_dir(root).unwrap();
15343        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15344    }
15345
15346    #[test]
15347    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15348        let path = Path::new("/definitely-missing-subc-cgroup");
15349        let mut command = Command::new("true");
15350        let error = apply_cgroup_placement(
15351            &mut command,
15352            &ModuleSpec {
15353                module_id: "broken-cgroup".to_string(),
15354                program: PathBuf::from("true"),
15355                args: Vec::new(),
15356                env: Vec::new(),
15357                reserved: false,
15358                reserved_prefixes: Vec::new(),
15359                protocol: ModuleProtocol::Subc,
15360                overlap: Default::default(),
15361            },
15362            path,
15363        )
15364        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15365        let reason = error.to_string();
15366
15367        assert!(
15368            matches!(error, SuperviseError::Cgroup { .. }),
15369            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15370        );
15371        assert!(
15372            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15373            "parent cgroup open failure must name cgroup.procs: {reason}"
15374        );
15375    }
15376
15377    #[tokio::test]
15378    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15379        let root = ScratchDir::new("supervisor-reap-cgroup");
15380        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15381        let placement = subc_cgroup::prepare_at(&root)
15382            .expect("prepare scratch cgroup root")
15383            .expect("scratch root has a cgroup.procs marker");
15384        let module_id = "reaped-module";
15385        let module = placement
15386            .module_path(module_id)
15387            .expect("create scratch module cgroup");
15388        let child = Command::new("true")
15389            .env("XDG_DATA_HOME", root.path())
15390            .env("XDG_RUNTIME_DIR", root.path())
15391            .env("XDG_CONFIG_HOME", root.path())
15392            .spawn()
15393            .expect("spawn short-lived child");
15394        let pid = child.id().expect("spawned child has pid");
15395        let mut child = SupervisedChild {
15396            child,
15397            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15398            protocol: ModuleProtocol::Subc,
15399            module_id: module_id.to_string(),
15400            cgroup_placement: Some(placement),
15401            stdout_pump: None,
15402            stderr_pump: None,
15403            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15404            spawned_at: tokio::time::Instant::now(),
15405            spawned_at_ms: 0,
15406            spawned_from: PathBuf::from("true"),
15407            launch_spec_digest: LaunchSpecDigest([0; 32]),
15408            spawned_file_identity: None,
15409            process_start_time: None,
15410            process_identity: None,
15411            pid,
15412            roster_guard: None,
15413            #[cfg(target_os = "macos")]
15414            privacy_exec: None,
15415            spawn_failure: None,
15416        };
15417
15418        child.wait().await.expect("reap short-lived child");
15419
15420        assert!(
15421            !module.exists(),
15422            "reaping the supervised child must remove its empty cgroup"
15423        );
15424    }
15425
15426    #[test]
15427    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15428        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15429        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15430        let placement = subc_cgroup::prepare_at(&root)
15431            .expect("prepare scratch cgroup root")
15432            .expect("scratch root has a cgroup.procs marker");
15433        let module = placement
15434            .module_path("surviving-module")
15435            .expect("create scratch module cgroup");
15436        fs::write(module.join("surviving-process"), b"still present")
15437            .expect("make scratch cgroup non-empty");
15438        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15439
15440        remove_module_cgroup(&placement, "surviving-module");
15441
15442        let logs = crate::router::test_log::captured_logs(&logs);
15443        assert!(
15444            module.exists(),
15445            "failed removal must leave the cgroup intact"
15446        );
15447        assert!(
15448            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15449                && logs.contains("surviving-module"),
15450            "best-effort removal must report the failure without returning it: {logs}"
15451        );
15452    }
15453
15454    #[test]
15455    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15456        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15457        let reason = SuperviseError::Spawn {
15458            program: PathBuf::from("/bin/true"),
15459            source: io::Error::from_raw_os_error(13),
15460            cgroup_path: Some(cgroup_path.clone()),
15461        }
15462        .to_string();
15463
15464        assert!(
15465            reason.contains(&cgroup_path.display().to_string()),
15466            "a pre_exec spawn failure must name the cgroup path: {reason}"
15467        );
15468    }
15469}
15470
15471#[cfg(test)]
15472mod spawn_subscriber_lag_tests {
15473    use super::*;
15474
15475    /// A subscriber whose connection stops draining is dropped once its frame
15476    /// channel fills. The client must learn that from a terminal Error frame
15477    /// after the frames already queued for it, not from a stream that simply
15478    /// goes quiet.
15479    #[tokio::test]
15480    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15481        let feed = SpawnEventFeed::default();
15482        feed.configure_incarnation("lag-incarnation".to_string());
15483        // A one-slot connection queue that nobody reads until the emits are
15484        // done: the forwarder parks on it and the subscriber channel fills.
15485        let (tx, mut rx) = mpsc::channel(1);
15486        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15487            .expect("subscribe");
15488        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15489        for index in 0..emitted {
15490            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15491            // Let the forwarder take what it can so the fill point is the
15492            // subscriber channel, not a scheduling accident.
15493            tokio::task::yield_now().await;
15494        }
15495        assert_eq!(
15496            feed.subscriber_count(),
15497            0,
15498            "the lagged subscriber must be removed"
15499        );
15500
15501        let mut data = Vec::new();
15502        let mut last = None;
15503        loop {
15504            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15505                .await
15506                .expect("the forwarder must finish once the subscriber is dropped");
15507            let Some(outbound) = next else { break };
15508            let frame = outbound.frame;
15509            if frame.header.ty == FrameType::StreamData {
15510                assert!(last.is_none(), "no data may follow the terminal frame");
15511                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15512                data.push(event.cursor.seq);
15513            } else {
15514                assert!(last.is_none(), "exactly one terminal frame");
15515                last = Some(frame);
15516            }
15517        }
15518        assert!(!data.is_empty(), "queued frames drain before the terminal");
15519        for pair in data.windows(2) {
15520            assert_eq!(
15521                pair[1],
15522                pair[0] + 1,
15523                "queued frames arrive dense and in order"
15524            );
15525        }
15526        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15527        assert_eq!(terminal.header.ty, FrameType::Error);
15528        assert_eq!(terminal.header.corr, 7);
15529        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15530        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15531        let detail = body.detail.expect("lagged error carries detail");
15532        assert_eq!(
15533            detail["first_undelivered_cursor"]["seq"],
15534            data.last().unwrap() + 1,
15535            "the named cursor is the first event the subscriber did not receive"
15536        );
15537        assert_eq!(
15538            detail["first_undelivered_cursor"]["daemon_incarnation"],
15539            "lag-incarnation"
15540        );
15541    }
15542}
15543
15544#[cfg(test)]
15545mod terminal_history_read_concurrency_tests {
15546    use super::*;
15547    use crate::terminal_journal::read_pause;
15548    use cortexkit_test_support::ScratchDir;
15549    use std::sync::mpsc as std_mpsc;
15550
15551    fn journaled_ring(
15552        journal: &Arc<crate::terminal_journal::TerminalJournal>,
15553    ) -> Arc<Mutex<TerminalRing>> {
15554        Arc::new(Mutex::new(
15555            TerminalRing::new(TerminalRingConfig::default(), 1)
15556                .with_journal(Some(Arc::clone(journal))),
15557        ))
15558    }
15559
15560    fn crash(at_ms: u64) -> ExitReport {
15561        ExitReport {
15562            kind: ExitKind::Crash,
15563            code: Some(1),
15564            signal: None,
15565            at_ms,
15566        }
15567    }
15568
15569    /// Record an exit on another thread and report whether it finished within
15570    /// `bound`. The recorder thread is left running if it did not.
15571    fn record_within(
15572        module_id: &'static str,
15573        ring: &Arc<Mutex<TerminalRing>>,
15574        at_ms: u64,
15575        bound: Duration,
15576    ) -> bool {
15577        let ring = Arc::clone(ring);
15578        let (done, done_rx) = std_mpsc::channel();
15579        std::thread::spawn(move || {
15580            record_terminal(
15581                module_id,
15582                &ring,
15583                &SpawnEventFeed::default(),
15584                &crash(at_ms),
15585                TerminalDisposition::Restarting,
15586            );
15587            let _ = done.send(());
15588        });
15589        done_rx.recv_timeout(bound).is_ok()
15590    }
15591
15592    /// A history read in progress must not hold the journal writer (which every
15593    /// module's exit recording needs) or the module's own ring. Exits recorded
15594    /// while the read is paused complete promptly; the paused read answers as of
15595    /// the moment it started, and the next read has each exit exactly once.
15596    #[test]
15597    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15598        let dir = ScratchDir::new("terminal-history-concurrent-read");
15599        let path = dir.join("terminals.jsonl");
15600        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15601            path.clone(),
15602            "daemon".into(),
15603        ));
15604        let reader_ring = journaled_ring(&journal);
15605        let other_ring = journaled_ring(&journal);
15606        assert!(record_within(
15607            "reader-module",
15608            &reader_ring,
15609            10,
15610            Duration::from_secs(5)
15611        ));
15612
15613        let (started, release) = read_pause::install(&path);
15614        let reading = {
15615            let ring = Arc::clone(&reader_ring);
15616            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15617        };
15618        started
15619            .recv_timeout(Duration::from_secs(5))
15620            .expect("the history read reached its pause");
15621
15622        let bound = Duration::from_secs(1);
15623        assert!(
15624            record_within("other-module", &other_ring, 20, bound),
15625            "another module's exit waited on a history read (journal writer held)"
15626        );
15627        assert!(
15628            record_within("reader-module", &reader_ring, 30, bound),
15629            "the read module's own exit waited on its history read (ring held)"
15630        );
15631
15632        drop(release);
15633        let paused = reading.join().unwrap();
15634        assert_eq!(
15635            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15636            vec![10],
15637            "an exit recorded after the read began lands in neither half of it"
15638        );
15639        assert_eq!(paused.journal_skipped_lines, 0);
15640        assert_eq!(paused.journal_read_errors, 0);
15641
15642        let after = durable_terminal_history_of(&reader_ring, "reader-module");
15643        assert_eq!(
15644            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15645            vec![10, 30],
15646            "the next read merges ring and journal with no duplicate"
15647        );
15648        assert_eq!(after.journal_skipped_lines, 0);
15649    }
15650}
15651
15652/// What a restart does with the exited process's output readers. These drive
15653/// the same settle functions the supervisor calls, with a reader the test
15654/// holds, so a reader that has not been scheduled by the bound is a controlled
15655/// input rather than something only a loaded machine produces.
15656#[cfg(test)]
15657mod stderr_settle_tests {
15658    use std::{
15659        future::Future,
15660        io,
15661        pin::Pin,
15662        sync::{Arc, Mutex},
15663        task::{Context, Poll},
15664        time::Duration,
15665    };
15666
15667    use tokio::{
15668        io::{AsyncRead, ReadBuf},
15669        sync::oneshot,
15670        time::Instant,
15671    };
15672
15673    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15674    use crate::stderr_tail::{
15675        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15676        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15677    };
15678
15679    const BOUND: Duration = Duration::from_millis(250);
15680
15681    /// Yields `before`, then stays pending until the gate is released, then
15682    /// yields `after` and reaches EOF. The bytes after the gate were written
15683    /// by a process that has already exited; only the reader is behind.
15684    struct HeldReader {
15685        before: Option<Vec<u8>>,
15686        gate: Option<oneshot::Receiver<()>>,
15687        after: io::Cursor<Vec<u8>>,
15688    }
15689
15690    impl AsyncRead for HeldReader {
15691        fn poll_read(
15692            mut self: Pin<&mut Self>,
15693            cx: &mut Context<'_>,
15694            buf: &mut ReadBuf<'_>,
15695        ) -> Poll<io::Result<()>> {
15696            if let Some(bytes) = self.before.take() {
15697                buf.put_slice(&bytes);
15698                return Poll::Ready(Ok(()));
15699            }
15700            if let Some(gate) = self.gate.as_mut() {
15701                match Pin::new(gate).poll(cx) {
15702                    Poll::Pending => return Poll::Pending,
15703                    Poll::Ready(_) => self.gate = None,
15704                }
15705            }
15706            Pin::new(&mut self.after).poll_read(cx, buf)
15707        }
15708    }
15709
15710    struct DiscardSink;
15711
15712    impl OutputSink for DiscardSink {
15713        fn write_line(&mut self, _line: &[u8]) {}
15714    }
15715
15716    fn line(text: &str) -> TailEntry {
15717        TailEntry::Line {
15718            text: text.to_string(),
15719            truncated: false,
15720            at_ms: None,
15721        }
15722    }
15723
15724    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15725        ring.lock().unwrap()
15726    }
15727
15728    /// Start a reader for a new process generation that delivers `before`
15729    /// immediately and `after` only once the returned sender fires (or is
15730    /// dropped).
15731    fn held_pump(
15732        ring: &Arc<Mutex<StderrRing>>,
15733        before: &str,
15734        after: &str,
15735    ) -> (StderrPump, oneshot::Sender<()>) {
15736        let generation = lock(ring).begin_process();
15737        let (release, gate) = oneshot::channel();
15738        let reader = HeldReader {
15739            before: Some(before.as_bytes().to_vec()),
15740            gate: Some(gate),
15741            after: io::Cursor::new(after.as_bytes().to_vec()),
15742        };
15743        let task = tokio::spawn(pump_stderr_to(
15744            reader,
15745            Arc::clone(ring),
15746            generation,
15747            DiscardSink,
15748        ));
15749        (StderrPump { task, generation }, release)
15750    }
15751
15752    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15753        for _ in 0..1000 {
15754            if done(&lock(ring)) {
15755                return;
15756            }
15757            tokio::time::sleep(Duration::from_millis(1)).await;
15758        }
15759        panic!(
15760            "ring never reached the expected state: {:?}",
15761            lock(ring).snapshot(None, None)
15762        );
15763    }
15764
15765    #[tokio::test(start_paused = true)]
15766    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15767        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15768        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15769
15770        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15771        let before_release = lock(&ring).snapshot(None, None);
15772        assert!(
15773            matches!(before_release.capture, CaptureState::Incomplete { .. }),
15774            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15775        );
15776
15777        // The restart: the next process starts and writes before the old
15778        // reader catches up.
15779        let next = lock(&ring).begin_process();
15780        lock(&ring).push_line_from(next, "next process booting");
15781        release.send(()).unwrap();
15782        wait_until(&ring, |ring| {
15783            ring.snapshot(None, None).capture == CaptureState::Captured
15784        })
15785        .await;
15786
15787        assert_eq!(
15788            untimed(lock(&ring).snapshot(None, None).entries),
15789            vec![
15790                line("booting"),
15791                line("config error: missing storage"),
15792                TailEntry::ProcessStart,
15793                line("next process booting"),
15794            ],
15795            "the crash's last line must survive a slow reader and stay in the crashed process's section"
15796        );
15797    }
15798
15799    #[tokio::test(start_paused = true)]
15800    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15801    ) {
15802        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15803        // `_held` is never fired: a descendant keeps the pipe open for the
15804        // whole test.
15805        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15806
15807        let started = Instant::now();
15808        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15809        assert_eq!(
15810            started.elapsed(),
15811            BOUND,
15812            "the restart must wait exactly the bound for a pipe that stays open, no longer"
15813        );
15814
15815        let next = lock(&ring).begin_process();
15816        lock(&ring).push_line_from(next, "next process booting");
15817        tokio::time::sleep(Duration::from_secs(60)).await;
15818
15819        let snapshot = lock(&ring).snapshot(None, None);
15820        match &snapshot.capture {
15821            CaptureState::Incomplete { reason } => assert!(
15822                reason.contains("had not reached EOF") && reason.contains("250ms"),
15823                "the reason must say what is missing and after how long: {reason}"
15824            ),
15825            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15826        }
15827        assert_eq!(
15828            untimed(snapshot.entries),
15829            vec![
15830                line("parent exiting"),
15831                TailEntry::ProcessStart,
15832                line("next process booting"),
15833            ]
15834        );
15835    }
15836
15837    #[tokio::test(start_paused = true)]
15838    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15839        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15840        let (pump, release) = held_pump(&ring, "one\n", "two\n");
15841        release.send(()).unwrap();
15842
15843        settle_stderr_pump("clean", &ring, pump, BOUND).await;
15844
15845        let snapshot = lock(&ring).snapshot(None, None);
15846        assert_eq!(snapshot.capture, CaptureState::Captured);
15847        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15848    }
15849
15850    #[tokio::test(start_paused = true)]
15851    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15852        const BURST: usize = 150;
15853        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15854        let path = capture.join("two-pipe.stderr.log");
15855        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15856        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15857        let generation = lock(&ring).begin_process();
15858        let stdout = (0..BURST)
15859            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15860            .collect::<String>();
15861        let stderr = (0..BURST)
15862            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15863            .collect::<String>();
15864        // Model a child that has written both bursts and exited. Hold the stdout
15865        // reader after one line and a partial prefix, as if pipe readiness or
15866        // capture-file writes had delayed it. The final line also needs EOF to
15867        // flush: neither complete nor partial buffered output may be discarded.
15868        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15869        let (release, gate) = oneshot::channel();
15870        let reader = HeldReader {
15871            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15872            gate: Some(gate),
15873            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15874        };
15875        let (finished, completion) = oneshot::channel();
15876        let stdout_sink = sink.clone();
15877        let pump = tokio::spawn(async move {
15878            pump_stdout_to(reader, stdout_sink).await;
15879            finished.send(()).unwrap();
15880        });
15881        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15882
15883        let started = Instant::now();
15884        settle_stdout_pump("two-pipe", pump, BOUND).await;
15885        assert_eq!(
15886            started.elapsed(),
15887            BOUND,
15888            "a late reader must not hold up restart"
15889        );
15890        assert_eq!(
15891            std::fs::read_to_string(&path).unwrap().lines().count(),
15892            BURST + 1
15893        );
15894
15895        release
15896            .send(())
15897            .expect("stdout reader must survive the exit drain bound");
15898        tokio::time::timeout(Duration::from_secs(1), completion)
15899            .await
15900            .expect("released stdout reader must reach EOF")
15901            .expect("stdout capture must finish at EOF, not be cancelled");
15902        let contents = std::fs::read_to_string(&path).unwrap();
15903        let mut lines = contents
15904            .lines()
15905            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15906            .collect::<Vec<_>>();
15907        let mut expected = (0..BURST)
15908            .flat_map(|i| {
15909                [
15910                    format!("out-{i:04}-{}", "o".repeat(64)),
15911                    format!("err-{i:04}-{}", "e".repeat(64)),
15912                ]
15913            })
15914            .collect::<Vec<_>>();
15915        lines.sort_unstable();
15916        expected.sort_unstable();
15917        assert_eq!(
15918            lines, expected,
15919            "both pipes must retain every complete line exactly once"
15920        );
15921    }
15922}
15923
15924/// Containment of a module's process tree (issue #109).
15925///
15926/// The behaviour these defend against is a module helper surviving its module:
15927/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
15928/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
15929/// compounds it.
15930///
15931/// They run against the SUPERVISOR rather than the job-object crate because the
15932/// claim is about teardown: a crate-level test proves a job can reap a tree, not
15933/// that the daemon's drain path reaches it.
15934///
15935/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
15936/// lane there is a separate containment path with its own tests.
15937#[cfg(all(test, windows))]
15938mod job_containment_tests {
15939    use super::*;
15940    use cortexkit_test_support::ScratchDir;
15941    use std::{
15942        path::{Path, PathBuf},
15943        sync::{Arc, Mutex},
15944        time::{Duration, Instant},
15945    };
15946
15947    /// The stub, expected beside this test executable.
15948    ///
15949    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
15950    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
15951    /// failure then reads as a broken test rather than an unbuilt dependency.
15952    fn stub_path() -> PathBuf {
15953        let mut path = std::env::current_exe().expect("current_exe available in tests");
15954        path.pop();
15955        path.pop();
15956        path.push("fake-aft-stub.exe");
15957        assert!(
15958            path.exists(),
15959            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15960             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15961            path.display()
15962        );
15963        path
15964    }
15965
15966    /// Poll for the grandchild pid the stub records, and parse it.
15967    fn read_grandchild_pid(path: &Path) -> u32 {
15968        let deadline = Instant::now() + Duration::from_secs(10);
15969        loop {
15970            if let Ok(contents) = std::fs::read_to_string(path) {
15971                if let Ok(pid) = contents.trim().parse() {
15972                    return pid;
15973                }
15974            }
15975            assert!(
15976                Instant::now() < deadline,
15977                "the stub never recorded a grandchild pid at {}",
15978                path.display()
15979            );
15980            std::thread::sleep(Duration::from_millis(10));
15981        }
15982    }
15983
15984    /// Everything one fixture run needs, so the two tests below differ in exactly
15985    /// one place: whether the child is contained.
15986    struct Fixture {
15987        _dir: ScratchDir,
15988        module_id: String,
15989        grandchild: u32,
15990        child: Option<SupervisedChild>,
15991        registry: Arc<Registry>,
15992        snapshot: Arc<Mutex<SupervisorSnapshot>>,
15993        terminal_ring: Arc<Mutex<TerminalRing>>,
15994        spawn_events: SpawnEventFeed,
15995    }
15996
15997    fn fixture(label: &str, module_id: &str) -> Fixture {
15998        let dir = ScratchDir::new(label);
15999        let pid_file = dir.join("grandchild.pid");
16000        let supervisor = Supervisor::new_for_test(
16001            Arc::new(Registry::default()),
16002            RestartPolicy::new(3, Duration::ZERO),
16003        );
16004        let runtime = supervisor.runtime_config();
16005        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16006        let spec = ModuleSpec {
16007            module_id: module_id.to_string(),
16008            program: stub_path(),
16009            // Zero args deliberately: a `--subc` argument would make the stub dial
16010            // a daemon that is not there, and the failure would land in the same
16011            // stderr ring this fixture exists to keep quiet.
16012            args: Vec::new(),
16013            env: vec![
16014                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16015                (
16016                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16017                    pid_file.display().to_string(),
16018                ),
16019            ],
16020            reserved: false,
16021            reserved_prefixes: Vec::new(),
16022            protocol: ModuleProtocol::Subc,
16023            overlap: Default::default(),
16024        };
16025        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16026            .expect("spawn the supervised fixture");
16027        let grandchild = read_grandchild_pid(&pid_file);
16028        Fixture {
16029            _dir: dir,
16030            module_id: module_id.to_string(),
16031            grandchild,
16032            child: Some(child),
16033            registry: Arc::new(Registry::default()),
16034            snapshot,
16035            terminal_ring: Arc::clone(&runtime.terminal_ring),
16036            spawn_events: SpawnEventFeed::default(),
16037        }
16038    }
16039
16040    impl Fixture {
16041        /// Drain through the supervisor's own teardown path.
16042        async fn drain(&mut self) {
16043            let child = self
16044                .child
16045                .take()
16046                .expect("the fixture child is still present");
16047            drain_child_to_state(
16048                &self.module_id,
16049                ModuleProtocol::Subc,
16050                // No forwarding table in this fixture, so nothing reaches the
16051                // child over a connection.
16052                StopNotice::NotSent,
16053                &self.registry,
16054                None,
16055                &self.snapshot,
16056                &self.terminal_ring,
16057                &self.spawn_events,
16058                child,
16059                Duration::from_millis(500),
16060                ModuleState::Stopped,
16061                Some(false),
16062            )
16063            .await
16064            .expect("drain the supervised fixture");
16065        }
16066    }
16067
16068    /// Teardown reaps the grandchild, not merely the direct child.
16069    ///
16070    /// This is the assertion the change exists for. Before containment the
16071    /// grandchild survived: it is a separate process, and `start_kill` is
16072    /// `TerminateProcess` scoped to one pid.
16073    #[tokio::test]
16074    async fn teardown_reaps_the_grandchild() {
16075        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16076        let grandchild = fixture.grandchild;
16077
16078        assert!(
16079            subc_jobobject::process_exists(grandchild),
16080            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16081        );
16082
16083        fixture.drain().await;
16084
16085        assert!(
16086            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16087            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16088        );
16089    }
16090
16091    /// The mutation control: with containment withheld, the grandchild survives
16092    /// the same kill.
16093    ///
16094    /// This is the defect reproduction from #109 — a direct-child kill reaches
16095    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16096    /// supervisor because `spawn_and_mark_running` now always contains on
16097    /// Windows, which is the point: there is no longer a path that spawns
16098    /// uncontained, so the control has to construct one.
16099    ///
16100    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16101    /// grandchild ever dies here, that test is passing for a reason unrelated to
16102    /// the job object and the containment claim is unproven.
16103    #[test]
16104    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16105        let dir = ScratchDir::new("teardown-uncontained");
16106        let pid_file = dir.join("grandchild.pid");
16107        let mut child = std::process::Command::new(stub_path())
16108            .env("FAKE_AFT_NEVER_CONNECT", "1")
16109            .env(
16110                "FAKE_AFT_GRANDCHILD_PID_FILE",
16111                pid_file.display().to_string(),
16112            )
16113            .stdin(std::process::Stdio::null())
16114            .stdout(std::process::Stdio::null())
16115            .stderr(std::process::Stdio::null())
16116            .spawn()
16117            .expect("spawn the uncontained fixture");
16118        let grandchild = read_grandchild_pid(&pid_file);
16119
16120        // Exactly what the pre-fix teardown did: kill the direct child.
16121        child.kill().expect("kill the direct child");
16122        let _ = child.wait();
16123
16124        assert!(
16125            subc_jobobject::process_exists(grandchild),
16126            "grandchild {grandchild} died with the direct child, so this control no longer \
16127             distinguishes contained from uncontained teardown and the regression test is \
16128             passing vacuously"
16129        );
16130
16131        // The orphan this control demonstrates is the leak the fix prevents, so
16132        // the control must not leave one behind.
16133        kill_tree(grandchild);
16134    }
16135
16136    /// Crash durability: closing the containment handle reaps the tree with no
16137    /// teardown code running at all.
16138    ///
16139    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16140    /// call anything — and it is why containment is a kernel property of the
16141    /// handle rather than a step in the drain. Discovered by getting the
16142    /// mutation control wrong: clearing `job` to "disable" containment instead
16143    /// killed the tree, which is the guarantee, not a mistake.
16144    #[tokio::test]
16145    async fn dropping_containment_reaps_the_grandchild() {
16146        let mut fixture = fixture("drop-containment", "tree-drop");
16147        let grandchild = fixture.grandchild;
16148
16149        assert!(subc_jobobject::process_exists(grandchild));
16150
16151        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16152        fixture.child.as_mut().expect("child present").job = None;
16153
16154        assert!(
16155            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16156            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16157             crash would leave the tree behind"
16158        );
16159    }
16160
16161    /// Kill a pid and its tree, then confirm it is gone.
16162    fn kill_tree(pid: u32) {
16163        let _ = std::process::Command::new("taskkill.exe")
16164            .args(["/PID", &pid.to_string(), "/T", "/F"])
16165            .stdin(std::process::Stdio::null())
16166            .stdout(std::process::Stdio::null())
16167            .stderr(std::process::Stdio::null())
16168            .status();
16169        assert!(
16170            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16171            "could not clean up grandchild {pid}"
16172        );
16173    }
16174}
16175
16176#[cfg(test)]
16177mod privacy_trampoline_configuration_tests {
16178    #[tokio::test]
16179    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16180    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16181        #[cfg(target_os = "macos")]
16182        {
16183            let supervisor = super::Supervisor::new(
16184                std::sync::Arc::new(crate::Registry::default()),
16185                super::RestartPolicy::default(),
16186            );
16187            let error = supervisor.spawn(spec()).unwrap_err();
16188            assert!(
16189                error
16190                    .to_string()
16191                    .contains("no privacy trampoline configured"),
16192                "{error}"
16193            );
16194        }
16195    }
16196
16197    #[tokio::test]
16198    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16199    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16200        #[cfg(target_os = "macos")]
16201        {
16202            let supervisor = super::Supervisor::new(
16203                std::sync::Arc::new(crate::Registry::default()),
16204                super::RestartPolicy::default(),
16205            )
16206            .with_privacy_trampoline(std::env::current_exe().unwrap());
16207            let error = supervisor.spawn(spec()).unwrap_err();
16208            assert!(
16209                error
16210                    .to_string()
16211                    .contains("binary does not implement the privacy trampoline protocol"),
16212                "{error}"
16213            );
16214        }
16215    }
16216
16217    #[cfg(target_os = "macos")]
16218    fn spec() -> super::ModuleSpec {
16219        super::ModuleSpec {
16220            module_id: "privacy-configuration".into(),
16221            program: "/bin/sleep".into(),
16222            args: vec!["30".into()],
16223            env: vec![],
16224            reserved: false,
16225            reserved_prefixes: vec![],
16226            protocol: subc_control::ModuleProtocol::None,
16227            overlap: super::ModuleOverlap::Exclusive,
16228        }
16229    }
16230}
16231
16232#[cfg(test)]
16233mod privacy_exec_boundary_tests {
16234    #[cfg(target_os = "macos")]
16235    use super::*;
16236    #[cfg(target_os = "macos")]
16237    use std::{
16238        io::{Read, Write},
16239        net::{TcpListener, TcpStream},
16240    };
16241
16242    /// Unit-test-only pause at the actual early image read, not at a later
16243    /// status read. Production supervisors never inspect this environment key.
16244    #[cfg(target_os = "macos")]
16245    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16246        if let Some((_, path)) = spec
16247            .env
16248            .iter()
16249            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16250        {
16251            let mut barrier = TcpStream::connect(path).unwrap();
16252            barrier
16253                .set_read_timeout(Some(Duration::from_secs(30)))
16254                .unwrap();
16255            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16256            let mut release = [0];
16257            barrier.read_exact(&mut release).unwrap();
16258            assert_eq!(&release, b"X");
16259        }
16260    }
16261
16262    #[cfg(target_os = "macos")]
16263    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16264        ModuleSpec {
16265            module_id: "privacy-boundary".into(),
16266            program: program.into(),
16267            args: args.iter().map(|arg| (*arg).into()).collect(),
16268            env: vec![],
16269            reserved: false,
16270            reserved_prefixes: vec![],
16271            protocol: ModuleProtocol::None,
16272            overlap: ModuleOverlap::Exclusive,
16273        }
16274    }
16275
16276    #[cfg(target_os = "macos")]
16277    async fn accept(listener: TcpListener) -> TcpStream {
16278        // Socket readiness, not elapsed time, establishes both pause points.
16279        let listener = tokio::net::TcpListener::from_std({
16280            listener.set_nonblocking(true).unwrap();
16281            listener
16282        })
16283        .unwrap();
16284        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16285            .await
16286            .unwrap()
16287            .unwrap();
16288        let stream = stream.into_std().unwrap();
16289        stream.set_nonblocking(false).unwrap();
16290        stream
16291            .set_read_timeout(Some(Duration::from_secs(30)))
16292            .unwrap();
16293        stream
16294    }
16295
16296    #[tokio::test]
16297    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16298    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16299        #[cfg(target_os = "macos")]
16300        {
16301            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16302            // Loopback sockets also work when the replay adapter's TMPDIR is
16303            // longer than Darwin's Unix-domain socket path limit.
16304            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16305            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16306            let record = root.join("live-children.json");
16307            let supervisor =
16308                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16309                    .with_live_children_record(&record);
16310            let runtime = supervisor.runtime_config();
16311            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16312            let mut spec = spec("/bin/sleep", &["30"]);
16313            spec.env = vec![
16314                (
16315                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16316                    exec_listener.local_addr().unwrap().to_string(),
16317                ),
16318                (
16319                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16320                    sample_listener.local_addr().unwrap().to_string(),
16321                ),
16322            ];
16323            let spawn = tokio::task::spawn_blocking(move || {
16324                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16325            });
16326            let mut sample = accept(sample_listener).await;
16327            let mut pid = [0; 4];
16328            sample.read_exact(&mut pid).unwrap();
16329            let pid = u32::from_ne_bytes(pid);
16330            let mut exec = accept(exec_listener).await;
16331            let mut ready = [0];
16332            exec.read_exact(&mut ready).unwrap();
16333            assert_eq!(&ready, b"R");
16334            // The early read is guaranteed to see a real, non-null trampoline
16335            // image: the fixture has reached its barrier and cannot exec yet.
16336            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16337            assert_eq!(
16338                observe_spawned_image(pid).unwrap().executable,
16339                Some(trampoline)
16340            );
16341            sample.write_all(b"X").unwrap();
16342            let mut child = spawn.await.unwrap();
16343            let early = crate::live_children::read_record(&record).unwrap();
16344            assert_eq!(early.len(), 1);
16345            assert_eq!(early[0].pid, pid);
16346            assert_eq!(
16347                early[0].executable, None,
16348                "unconfirmed trampoline image entered the roster"
16349            );
16350            assert!(child.report_ready.get().is_none());
16351            // The barrier's duration is unrelated to the production five-second
16352            // exec budget. Start the test's confirmation budget upon release.
16353            child.privacy_exec.as_mut().unwrap().deadline =
16354                tokio::time::Instant::now() + Duration::from_secs(30);
16355            exec.write_all(b"X").unwrap();
16356            child.confirm_privacy_exec().await;
16357            assert_eq!(child.spawn_failure, None);
16358            assert!(child.report_ready.get().is_some());
16359            let confirmed = crate::live_children::read_record(&record).unwrap();
16360            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16361            assert_ne!(module, trampoline);
16362            assert_eq!(confirmed[0].executable, Some(module.into()));
16363            child.start_kill().unwrap();
16364            child.wait().await.unwrap();
16365            child.release_roster();
16366        }
16367    }
16368
16369    #[tokio::test]
16370    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16371    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16372        #[cfg(target_os = "macos")]
16373        {
16374            let registry = Arc::new(Registry::default());
16375            let policy = RestartPolicy::new(0, Duration::ZERO);
16376            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16377            let runtime = supervisor.runtime_config();
16378            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16379            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16380            // Drive spawn and confirmation separately instead of starting the
16381            // monitor. WNOWAIT observes a real exit without consuming its status,
16382            // so confirmation's first try_wait must take the already-exited arm.
16383            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16384            let pid = child.pid;
16385            tokio::task::spawn_blocking(move || {
16386                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16387            })
16388            .await
16389            .unwrap()
16390            .unwrap();
16391            child.privacy_exec.as_mut().unwrap().deadline =
16392                tokio::time::Instant::now() + Duration::from_secs(30);
16393            let status = child.wait().await.unwrap();
16394            assert_eq!(status.code(), Some(121));
16395            assert!(child.privacy_exec.is_none());
16396            assert!(
16397                child.report_ready.get().is_none(),
16398                "an exited module must not publish a live pid"
16399            );
16400            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16401            on_child_exit(
16402                &spec,
16403                policy,
16404                &registry,
16405                &snapshot,
16406                &runtime.terminal_ring,
16407                &runtime.spawn_events,
16408                &runtime.child_roster,
16409                report,
16410            )
16411            .await;
16412            let state = lock_snapshot(&snapshot).unwrap();
16413            assert_eq!(state.state, ModuleState::Failed);
16414            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16415            assert_eq!(state.reported_pid(), None);
16416            drop(state);
16417            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16418            assert_eq!(history.entries.len(), 1);
16419            let terminal = &history.entries[0];
16420            assert_eq!(terminal.exit_code, Some(121));
16421            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16422            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16423            assert_eq!(
16424                terminal.disposition_detail.as_deref(),
16425                Some(policy.budget_exhausted_detail().as_str()),
16426                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16427            );
16428            assert_eq!(child.spawn_failure, None);
16429            child.release_roster();
16430        }
16431    }
16432}
16433
16434/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16435/// descriptor 3, without an environment copy. The shell records the nonce
16436/// and its environment after exec so these tests observe the real handover.
16437#[cfg(all(test, unix))]
16438mod launch_nonce_descriptor_tests {
16439    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16440    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16441    use cortexkit_test_support::ScratchDir;
16442    use std::{
16443        path::PathBuf,
16444        sync::{Arc, Mutex},
16445        time::{Duration, Instant},
16446    };
16447
16448    async fn probe(role: super::SpawnRole) {
16449        let scratch = ScratchDir::new("launch-nonce-descriptor");
16450        let fd_copy = scratch.join("from-descriptor");
16451        let env_copy = scratch.join("environment");
16452        let script = format!(
16453            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16454            fd = fd_copy.display(), env = env_copy.display(),
16455        );
16456        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16457        let spec = ModuleSpec {
16458            module_id: "nonce-descriptor-probe".to_string(),
16459            program: PathBuf::from("/bin/sh"),
16460            args: vec!["-c".to_string(), script],
16461            env: vec![
16462                xdg("XDG_DATA_HOME"),
16463                xdg("XDG_RUNTIME_DIR"),
16464                xdg("XDG_CONFIG_HOME"),
16465                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16466            ],
16467            reserved: true,
16468            reserved_prefixes: Vec::new(),
16469            protocol: ModuleProtocol::Subc,
16470            overlap: Default::default(),
16471        };
16472        let handle = SupervisorHandle::new();
16473        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16474        let roster = ChildRoster::default();
16475        #[cfg(target_os = "macos")]
16476        {
16477            let path = super::test_privacy_trampoline();
16478            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16479        }
16480        let child = super::spawn_child_in_slot(
16481            &spec,
16482            None,
16483            Some(&handle),
16484            &ring,
16485            None,
16486            &roster,
16487            #[cfg(target_os = "linux")]
16488            None,
16489            role,
16490            matches!(role, super::SpawnRole::SwapCandidate),
16491        )
16492        .expect("spawn probe");
16493        let deadline = Instant::now() + Duration::from_secs(10);
16494        while !(fd_copy.exists() && env_copy.exists()) {
16495            assert!(Instant::now() < deadline, "probe never wrote its copies");
16496            tokio::time::sleep(Duration::from_millis(20)).await;
16497        }
16498        let nonce = std::fs::read_to_string(fd_copy).unwrap();
16499        assert!(!nonce.is_empty());
16500        let environment = std::fs::read_to_string(env_copy).unwrap();
16501        assert!(environment
16502            .lines()
16503            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16504        let copy = environment
16505            .lines()
16506            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16507        assert_eq!(
16508            copy, None,
16509            "Unix children must never receive the environment nonce"
16510        );
16511        if matches!(role, super::SpawnRole::Plain) {
16512            assert_eq!(
16513                handle.spawn_nonce(&spec.module_id).as_deref(),
16514                Some(nonce.as_str())
16515            );
16516        }
16517        drop(child);
16518    }
16519
16520    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16521    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16522        probe(super::SpawnRole::Plain).await;
16523    }
16524
16525    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16526    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16527        probe(super::SpawnRole::SwapCandidate).await;
16528    }
16529}
16530
16531#[cfg(all(test, target_os = "linux"))]
16532mod cgroup_containment_tests {
16533    use super::*;
16534    use cortexkit_test_support::ScratchDir;
16535
16536    fn running(pid: u32) -> bool {
16537        // An orphan can remain a zombie until the container init reaps it.
16538        std::fs::read_to_string(format!("/proc/{pid}/stat"))
16539            .ok()
16540            .and_then(|stat| {
16541                stat.rsplit_once(") ")
16542                    .map(|(_, rest)| rest.starts_with('Z'))
16543            })
16544            .is_some_and(|zombie| !zombie)
16545    }
16546
16547    #[tokio::test]
16548    async fn linux_teardown_reaps_the_grandchild() {
16549        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16550    }
16551
16552    #[tokio::test]
16553    async fn linux_shutdown_straggler_reaps_the_grandchild() {
16554        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16555    }
16556
16557    async fn teardown_tree(test_name: &str, shutdown: bool) {
16558        let dir = ScratchDir::new(test_name);
16559        let root = PathBuf::from(format!(
16560            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16561            std::process::id(),
16562            unix_ms_now()
16563        ));
16564        if let Err(error) = std::fs::create_dir(&root) {
16565            assert!(
16566                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16567                "required cgroup test cannot execute: {error}"
16568            );
16569            eprintln!(
16570                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16571                root.display()
16572            );
16573            return;
16574        }
16575        let placement = subc_cgroup::prepare_at(&root)
16576            .expect("prepare isolated kernel cgroup")
16577            .expect("isolated cgroup is delegated");
16578        let module_id = "tree-teardown";
16579        let module = placement
16580            .module_path(module_id)
16581            .expect("create isolated module cgroup");
16582        if !module.join("cgroup.kill").exists() {
16583            std::fs::remove_dir(&module).unwrap();
16584            std::fs::remove_dir(root.join("subc-modules")).unwrap();
16585            std::fs::remove_dir(&root).unwrap();
16586            assert!(
16587                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16588                "required cgroup.kill interface unavailable"
16589            );
16590            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16591            return;
16592        }
16593        let supervisor = Supervisor::new_for_test(
16594            Arc::new(Registry::default()),
16595            RestartPolicy::new(3, Duration::ZERO),
16596        )
16597        .with_cgroup_placement(Some(placement));
16598        let mut runtime = supervisor.runtime_config();
16599        runtime.child_roster = runtime
16600            .child_roster
16601            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16602        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16603        let pid_file = dir.join("grandchild.pid");
16604        let spec = ModuleSpec {
16605            module_id: module_id.to_string(),
16606            program: PathBuf::from("/bin/sh"),
16607            args: vec![
16608                "-c".into(),
16609                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16610                "fixture".into(),
16611                pid_file.display().to_string(),
16612            ],
16613            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16614                .into_iter()
16615                .map(|key| (key.to_string(), dir.display().to_string()))
16616                .collect(),
16617            reserved: false,
16618            reserved_prefixes: Vec::new(),
16619            protocol: ModuleProtocol::None,
16620            overlap: Default::default(),
16621        };
16622        let child =
16623            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16624        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16625        let grandchild: u32 = loop {
16626            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16627                if let Ok(pid) = contents.trim().parse() {
16628                    break pid;
16629                }
16630            }
16631            assert!(
16632                tokio::time::Instant::now() < deadline,
16633                "grandchild pid was not recorded"
16634            );
16635            tokio::time::sleep(Duration::from_millis(10)).await;
16636        };
16637        assert!(
16638            running(grandchild),
16639            "grandchild must be alive before teardown"
16640        );
16641        if shutdown {
16642            let mut child = child;
16643            crate::child_roster::end_children_for_daemon_shutdown(
16644                &runtime.child_roster,
16645                false,
16646                std::future::pending(),
16647            )
16648            .await;
16649            child.wait().await.expect("reap shutdown straggler");
16650        } else {
16651            drain_child_to_state(
16652                module_id,
16653                ModuleProtocol::None,
16654                StopNotice::NotSent,
16655                &Registry::default(),
16656                None,
16657                &snapshot,
16658                &runtime.terminal_ring,
16659                &SpawnEventFeed::default(),
16660                child,
16661                Duration::from_millis(100),
16662                ModuleState::Stopped,
16663                Some(false),
16664            )
16665            .await
16666            .expect("real supervisor teardown");
16667        }
16668        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16669        while running(grandchild) && tokio::time::Instant::now() < deadline {
16670            tokio::time::sleep(Duration::from_millis(10)).await;
16671        }
16672        let survived = running(grandchild);
16673        // Kill a surviving grandchild so a failed test does not leave it behind.
16674        if survived {
16675            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16676            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16677            tokio::time::sleep(Duration::from_millis(100)).await;
16678        }
16679        if module.exists() {
16680            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16681        }
16682        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16683        std::fs::remove_dir(&root).unwrap();
16684        assert!(
16685            !survived,
16686            "grandchild {grandchild} outlived module teardown"
16687        );
16688        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16689    }
16690}