Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    /// Conservative monotonic bound sampled before `command.spawn()`, not when
362    /// identity recording finishes or the spawn generation is published.
363    spawned_at: Instant,
364    spawned_at_ms: u64,
365    spawned_from: PathBuf,
366    launch_spec_digest: LaunchSpecDigest,
367    spawned_file_identity: Option<SpawnedFileIdentity>,
368    process_start_time: Option<u64>,
369    process_identity: Option<ProcessIdentity>,
370    pid: u32,
371    /// This process's entry in the daemon's child roster, released when the
372    /// process is reaped or this handle is dropped.
373    roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377    fn id(&self) -> Option<u32> {
378        Some(self.pid)
379    }
380
381    fn process_identity(&self) -> Option<ProcessIdentity> {
382        self.process_identity
383    }
384
385    async fn wait(&mut self) -> io::Result<ExitStatus> {
386        #[cfg(target_os = "macos")]
387        self.confirm_privacy_exec().await;
388        // The roster entry is NOT released here. A daemon shutdown waits for the
389        // roster to empty and then exits the process, so releasing at the reap
390        // let it exit before the exit handler wrote this child's terminal record
391        // (the stderr drain and snapshot update sit in between), and the
392        // shutdown's own `daemon_shutdown` record was intermittently lost. The
393        // caller releases it after recording the exit (`release_roster`), and
394        // dropping the handle releases it too.
395        let result = self.child.wait().await;
396        #[cfg(target_os = "linux")]
397        if result.is_ok() {
398            if let Some(placement) = self.cgroup_placement.as_ref() {
399                cleanup_reaped_cgroup(placement, &self.module_id).await;
400                // Keep ownership while awaiting kernel population changes: a
401                // drain timeout may cancel this wait and then escalate/reap.
402                self.cgroup_placement = None;
403            }
404        }
405        result
406    }
407
408    #[cfg(target_os = "macos")]
409    async fn confirm_privacy_exec(&mut self) {
410        let Some(pending) = &mut self.privacy_exec else {
411            return;
412        };
413        let result = tokio::time::timeout_at(pending.deadline, async {
414            let mut record = Vec::new();
415            loop {
416                let mut ready = pending.reader.readable().await?;
417                let read = ready.try_io(|reader| {
418                    use std::io::Read;
419                    let mut reader = reader.get_ref();
420                    let mut buffer = [0u8; 256];
421                    reader.read(&mut buffer).map(|count| (count, buffer))
422                });
423                match read {
424                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425                    Ok(Ok((count, buffer))) => {
426                        if record.len() + count > 1024 {
427                            return Err(io::Error::other(
428                                "privacy exec refusal record is too long",
429                            ));
430                        }
431                        record.extend_from_slice(&buffer[..count]);
432                    }
433                    Ok(Err(error)) => return Err(error),
434                    Err(_) => continue,
435                }
436            }
437        })
438        .await;
439        #[cfg(test)]
440        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441            use tokio::io::{AsyncReadExt, AsyncWriteExt};
442            if stream.is_none() {
443                *stream = Some(
444                    tokio::net::TcpStream::connect(address.as_str())
445                        .await
446                        .unwrap(),
447                );
448                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449                stream
450                    .as_mut()
451                    .unwrap()
452                    .write_all(&self.pid.to_ne_bytes())
453                    .await
454                    .unwrap();
455            }
456            let mut reply = [0];
457            stream
458                .as_mut()
459                .unwrap()
460                .read_exact(&mut reply)
461                .await
462                .unwrap();
463            pending.confirmation_pause = None;
464            if &reply == b"F" {
465                Ok(Err(io::Error::other("injected late exec refusal")))
466            } else {
467                assert_eq!(&reply, b"X");
468                result
469            }
470        } else {
471            result
472        };
473        // Keep the reader in self across await: select cancellation must not
474        // discard the handshake or reset its original five-second deadline.
475        let pending = self.privacy_exec.as_ref().expect("pending exec");
476        let cause = match result {
477            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478            Ok(Err(error)) => Some(format!(
479                "privacy identity exec acknowledgement failed: {error}"
480            )),
481            Ok(Ok(record)) if !record.is_empty() => Some(
482                std::str::from_utf8(&record)
483                    .ok()
484                    .and_then(|record| {
485                        record
486                            .trim()
487                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488                    })
489                    .filter(|cause| !cause.is_empty())
490                    .unwrap_or("invalid privacy exec refusal record")
491                    .to_string(),
492            ),
493            Ok(Ok(_)) => match self.child.try_wait() {
494                // Empty EOF is the exec acknowledgement. A real module may exit
495                // immediately, including with a reserved trampoline status; no
496                // image is admitted, and its ordinary exit contract stays intact.
497                Ok(Some(_status)) => None,
498                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499                Ok(None) => {
500                    let image = observe_spawned_image(self.pid);
501                    if let Some(image) = image.filter(|image| {
502                        image.executable.is_some()
503                            && image.executable != pending.trampoline
504                            && (image.executable == pending.expected || pending.script)
505                    }) {
506                        if let Err(error) = self
507                            .roster_guard
508                            .as_ref()
509                            .map(|guard| guard.confirm_executable(image))
510                            .transpose()
511                        {
512                            Some(format!(
513                                "could not durably record confirmed module image: {error}"
514                            ))
515                        } else {
516                            let _ = self.report_ready.set(());
517                            info!(module_id = %pending.module_id, pid = self.pid,
518                                "module spawned with own privacy identity (responsibility disclaimed)");
519                            None
520                        }
521                    } else if image.is_none()
522                        || image.is_some_and(|image| image.executable.is_none())
523                    {
524                        // A process can exit between try_wait and the kernel
525                        // image read. Empty EOF already acknowledged exec, so
526                        // preserve that module's ordinary exit rather than
527                        // mislabel a disappearing image as trampoline refusal.
528                        // Keep pending in self across await so cancellation does
529                        // not discard validation or reset its original deadline.
530                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531                            Ok(Ok(_status)) => None,
532                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534                        }
535                    } else {
536                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537                    }
538                }
539            },
540        };
541        let pending = self.privacy_exec.take().expect("pending exec");
542        // Exec confirmation describes launch success, not roster retirement.
543        // Keep even a reaped child recorded until its terminal history exists.
544        self.exec.0.finish(if cause.is_some() {
545            crate::activity::ExecState::Refused
546        } else {
547            crate::activity::ExecState::CommittedExec
548        });
549        if let Some(cause) = cause {
550            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551            self.spawn_failure = Some(cause);
552            // No image is admitted on failure. Reach the entire fresh process
553            // group, including a module which spawned a helper before refusal.
554            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556            }
557            let _ = self.child.start_kill();
558        }
559    }
560
561    /// Releases this child's daemon-shutdown roster entry once its exit has
562    /// been recorded. The pid is already reaped and free for reuse, so the
563    /// entry must not outlive the record any longer than that.
564    fn release_roster(&mut self) {
565        self.roster_guard = None;
566    }
567
568    /// Kill the child and, where containment is available, its process tree.
569    ///
570    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
571    /// helper process leaked the helper — the Synapse embedding module's CUDA
572    /// worker holds the GPU allocation, so the leak cost VRAM until the next
573    /// restart of something else. Terminating the job reaches grandchildren that
574    /// a tree walk cannot, including one whose parent has already exited and
575    /// been reparented away.
576    ///
577    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
578    /// direct-child kill still decides the outcome, so containment can never
579    /// change whether a module is reported as stopped.
580    fn start_kill(&mut self) -> io::Result<()> {
581        #[cfg(windows)]
582        if let Some(job) = &self.job {
583            if let Err(error) = job.terminate() {
584                debug!(
585                    error = %error,
586                    "job termination failed; the direct-child kill still owns the outcome"
587                );
588            }
589        }
590        #[cfg(target_os = "linux")]
591        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592        self.child.start_kill()
593    }
594
595    async fn drain_stderr(&mut self, module_id: &str) {
596        if let Some(pump) = self.stdout_pump.take() {
597            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598        }
599
600        let Some(pump) = self.stderr_pump.take() else {
601            return;
602        };
603        settle_stderr_pump(
604            module_id,
605            &self.stderr_ring,
606            pump,
607            STDERR_PUMP_DRAIN_TIMEOUT,
608        )
609        .await;
610    }
611}
612
613/// Bound the restart's wait, not the lifetime of the exited child's stdout
614/// reader. Slow pipe reads or capture writes must not turn the bound into lost
615/// output; dropping the handle lets the reader finish at EOF, like stderr.
616async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617    match timeout(bound, &mut pump).await {
618        Ok(Ok(())) => {}
619        Ok(Err(error)) => {
620            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621        }
622        Err(_) => {
623            drop(pump);
624            warn!(
625                module_id,
626                waited = ?bound,
627                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628            );
629        }
630    }
631}
632
633/// The reader task for one process's stderr, with the ring generation its
634/// lines are attributed to.
635struct StderrPump {
636    task: JoinHandle<()>,
637    generation: u64,
638}
639
640/// Retire an exited process's stderr reader and wait up to `bound` for it to
641/// reach EOF. A reader still running at the bound is detached, not stopped: it
642/// keeps filling the exited process's section of the ring until its pipe
643/// closes, and the tail reads `Incomplete` until then. See
644/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
645async fn settle_stderr_pump(
646    module_id: &str,
647    ring: &Arc<Mutex<StderrRing>>,
648    pump: StderrPump,
649    bound: Duration,
650) {
651    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652    let StderrPump {
653        mut task,
654        generation,
655    } = pump;
656    lock().retire_pump(generation);
657    match timeout(bound, &mut task).await {
658        Ok(Ok(())) => {}
659        Ok(Err(err)) => {
660            let mut ring = lock();
661            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662            ring.finish_pump(generation);
663            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664        }
665        Err(_) => {
666            // Dropping the handle detaches the task; it ends at EOF on its pipe.
667            drop(task);
668            lock().mark_pump_late(
669                generation,
670                format!(
671                    "stderr of the exited process had not reached EOF {bound:?} after it was \
672                     retired (a descendant may still hold the pipe open); lines it still \
673                     writes are kept in that process's section"
674                ),
675            );
676            warn!(
677                module_id,
678                waited = ?bound,
679                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680            );
681        }
682    }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687    EVENTS.get_or_init(|| {
688        let (sender, _receiver) = watch::channel(0);
689        sender
690    })
691}
692
693pub(crate) fn notify_registration_release() {
694    let events = registration_release_events();
695    let next_generation = (*events.borrow()).wrapping_add(1);
696    events.send_replace(next_generation);
697}
698
699/// How to launch one singleton module process.
700#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702    pub module_id: String,
703    pub program: PathBuf,
704    pub args: Vec<String>,
705    pub env: Vec<(String, String)>,
706    /// When true this is a reserved module: each spawn gets a fresh one-time launch
707    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
708    /// process can register this module_id (a security-boundary module like the
709    /// credential vault must not be impersonable while it is down/restarting).
710    pub reserved: bool,
711    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
712    /// Prefixes come from daemon config and must end in `:` before they reach the
713    /// supervisor; the owner module's current spawn nonce authorizes claims under
714    /// each prefix.
715    pub reserved_prefixes: Vec<String>,
716    /// The wire protocol this module speaks, as DECLARED in daemon config.
717    ///
718    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
719    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
720    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
721    /// and NO launch nonce, and a clean exit the daemon did not request is
722    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
723    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
724    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
725    /// because a process ignores an environment variable it does not read.
726    ///
727    /// The argument is the part that cannot be "harmless to a process that
728    /// ignores it": a stock binary exits on an unknown flag before it listens
729    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
730    /// first conformance run against this mode found it. The nonce is withheld
731    /// because a process that will never present it gains nothing from holding
732    /// it, and a secret in the environment of a process that does not need it is
733    /// a leak surface for no benefit.
734    pub protocol: ModuleProtocol,
735    /// Whether two processes of this module may run at once, which is what a
736    /// blue/green swap does for the length of its overlap. Declared in daemon
737    /// config because the daemon must be able to answer it while the module is
738    /// down, and so a module cannot talk itself into it after registering.
739    pub overlap: ModuleOverlap,
740}
741
742/// Only the configured launch inputs enter this value; spawned connection paths,
743/// nonces and inherited environment never do. Its bytes remain daemon-private.
744#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749        f.write_str("LaunchSpecDigest([redacted])")
750    }
751}
752
753impl LaunchSpecDigest {
754    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755        fn field(hash: &mut Sha256, bytes: &[u8]) {
756            hash.update((bytes.len() as u64).to_le_bytes());
757            hash.update(bytes);
758        }
759        let mut hash = Sha256::new();
760        if let Some(program) = spec.program.to_str() {
761            field(&mut hash, b"utf8-program");
762            field(&mut hash, program.as_bytes());
763        } else {
764            // The daemon config is UTF-8, but in-process callers may provide
765            // native paths: lossy conversion would collapse distinct programs.
766            #[cfg(unix)]
767            {
768                use std::os::unix::ffi::OsStrExt;
769                field(&mut hash, b"unix-program");
770                field(&mut hash, spec.program.as_os_str().as_bytes());
771            }
772            #[cfg(windows)]
773            {
774                use std::os::windows::ffi::OsStrExt;
775                field(&mut hash, b"windows-program");
776                let bytes = spec
777                    .program
778                    .as_os_str()
779                    .encode_wide()
780                    .flat_map(u16::to_le_bytes)
781                    .collect::<Vec<_>>();
782                field(&mut hash, &bytes);
783            }
784            #[cfg(not(any(unix, windows)))]
785            field(&mut hash, spec.program.to_string_lossy().as_bytes());
786        }
787        hash.update((spec.args.len() as u64).to_le_bytes());
788        for arg in &spec.args {
789            field(&mut hash, arg.as_bytes());
790        }
791        let mut env = spec.env.iter().collect::<Vec<_>>();
792        env.sort_unstable();
793        hash.update((env.len() as u64).to_le_bytes());
794        for (key, value) in env {
795            field(&mut hash, key.as_bytes());
796            field(&mut hash, value.as_bytes());
797        }
798        field(
799            &mut hash,
800            match spec.protocol {
801                ModuleProtocol::Subc => b"subc",
802                ModuleProtocol::None => b"none",
803            },
804        );
805        Self(hash.finalize().into())
806    }
807}
808
809/// Whether a module tolerates a second process of itself running alongside.
810///
811/// Most modules are single-writer on their store (a WAL, a capture log, a
812/// resident index behind a writer barrier), and two processes on one store
813/// corrupt it. So a swap, which overlaps the old and new process by design,
814/// is refused unless the module's config opts in with `overlap: "safe"`.
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817    /// Never run two processes of this module at once. The default.
818    #[default]
819    Exclusive,
820    /// The module has said a second process of itself is harmless for the
821    /// length of a swap.
822    ///
823    /// Declare it only if a second instance can run for a few seconds without
824    /// touching ANY single-writer store: every database, WAL, index, projector
825    /// and scheduled job the module owns. A lease on part of that state is not
826    /// enough. broca's session lease guards WAL appends while its run index, its
827    /// store projector and its archive fold timer (which unlinks live WAL files)
828    /// stay single-writer, so broca is exclusive despite holding a lease. The
829    /// refusal only fires after this has been decided, so the decision is the
830    /// check.
831    Safe,
832}
833
834impl ModuleOverlap {
835    pub fn as_str(self) -> &'static str {
836        match self {
837            Self::Exclusive => "exclusive",
838            Self::Safe => "safe",
839        }
840    }
841}
842
843/// Environment variable telling a spawned module which case it was started
844/// for, before it sends HELLO. Only a swap candidate carries it, as
845/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
846///
847/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
848/// longer because nobody waits on it, while a plain restart must flip ready
849/// quickly because callers see `module_warming` until it does. Absence means
850/// plain restart, the safe reading. The daemon trusts nothing about it; the
851/// candidate is proven by its launch nonce at HELLO.
852pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
854pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855/// How long a swap waits for its candidate to register and declare itself
856/// ready when the operator does not say. A module warming as a swap candidate
857/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
858/// daemon allows that plus time to start the process and send HELLO.
859pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861/// Bounded restart policy for crash exits.
862///
863/// `max_restarts` is the number of replacement processes allowed after the
864/// initial spawn WITHIN `window`. After that many crash restarts inside one
865/// window the module enters [`ModuleState::Failed`] and the supervisor stops
866/// the crash loop.
867///
868/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
869/// and that only survived because crashes were rare: a module that crashed
870/// three times across a week was disabled forever by crashes that had nothing
871/// to do with each other. That stopped being survivable once modules began
872/// exiting non-zero whenever the daemon's connection to them drops, because
873/// then every daemon-side connection drop spends a unit of the same budget and
874/// one flappy hour permanently stops a healthy module. Restarts older than
875/// `window` release their slot, so a module that crashed twice yesterday has a
876/// full budget today, while a genuine crash loop -- which is fast by
877/// definition -- still reaches the cap and stops.
878#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880    pub max_restarts: u32,
881    /// Base delay before a crash replacement. The actual delay escalates with
882    /// the number of recent crash replacements and is capped by `max_backoff`.
883    pub backoff: Duration,
884    /// Maximum delay before a crash replacement.
885    pub max_backoff: Duration,
886    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
887    /// budget effectively infinite (nothing is ever in-window), which is why
888    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
889    pub window: Duration,
890}
891
892impl RestartPolicy {
893    /// A policy with the default crash window. Callers that care about the
894    /// window say so with [`Self::with_window`]; the ones that do not are
895    /// asking for the standard rate limit, not for no limit.
896    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897        Self {
898            max_restarts,
899            backoff,
900            max_backoff: DEFAULT_MAX_BACKOFF,
901            window: DEFAULT_RESTART_WINDOW,
902        }
903    }
904
905    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906        self.max_backoff = max_backoff;
907        self
908    }
909
910    pub fn with_window(mut self, window: Duration) -> Self {
911        self.window = window;
912        self
913    }
914
915    /// Calculate the capped exponential delay for the next crash replacement.
916    /// `restart_in_window` is zero for the first replacement after an operator
917    /// action (restart, reload, re-enable) cleared the crash ring, or after all
918    /// older crash replacements have aged out of the window.
919    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920        if self.backoff.is_zero() || self.max_backoff.is_zero() {
921            return Duration::ZERO;
922        }
923
924        let mut delay = self.backoff;
925        for _ in 0..restart_in_window {
926            if delay >= self.max_backoff {
927                return self.max_backoff;
928            }
929            delay = delay
930                .checked_mul(10)
931                .unwrap_or(self.max_backoff)
932                .min(self.max_backoff);
933        }
934        delay.min(self.max_backoff)
935    }
936
937    /// The one sentence that explains a budget-exhausted stop, used for both the
938    /// log line and the terminal record so the two cannot drift. It names the
939    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
940    /// exactly what this budget is not.
941    fn budget_exhausted_detail(&self) -> String {
942        format!(
943            "crash budget exhausted: max_restarts={} within window_secs={}",
944            self.max_restarts,
945            self.window.as_secs()
946        )
947    }
948}
949
950impl Default for RestartPolicy {
951    fn default() -> Self {
952        Self {
953            max_restarts: DEFAULT_MAX_RESTARTS,
954            backoff: DEFAULT_BACKOFF,
955            max_backoff: DEFAULT_MAX_BACKOFF,
956            window: DEFAULT_RESTART_WINDOW,
957        }
958    }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963    restart_in_window: u32,
964    delay: Duration,
965}
966
967/// Whether the daemon itself will bring this module back after the exit being
968/// handled: it is enabled AND its in-window crash restarts are below the cap.
969///
970/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
971/// the window are dropped here rather than by a timer, so the count is right
972/// the moment somebody asks and no bookkeeping runs for idle modules.
973fn daemon_will_restart(
974    state: &mut SupervisorSnapshot,
975    policy: &RestartPolicy,
976    now: Instant,
977) -> bool {
978    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988    Report,
989    Restart,
990    Alert,
991}
992
993impl fmt::Display for HealthAction {
994    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995        f.write_str(match self {
996            Self::Report => "report",
997            Self::Restart => "restart",
998            Self::Alert => "alert",
999        })
1000    }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005    /// Optional loopback HTTP endpoint for a managed non-wire process.
1006    /// Changing it applies live on rescan; the process protocol changes only
1007    /// at its next spawn.
1008    pub http: Option<String>,
1009    pub cadence: Duration,
1010    pub deadline: Duration,
1011    pub failure_threshold: u32,
1012    pub on_degraded: HealthAction,
1013    pub on_failing: HealthAction,
1014    pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018    fn default() -> Self {
1019        Self {
1020            http: None,
1021            cadence: DEFAULT_HEALTH_CADENCE,
1022            deadline: DEFAULT_HEALTH_DEADLINE,
1023            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024            on_degraded: HealthAction::Report,
1025            on_failing: HealthAction::Report,
1026            critical: false,
1027        }
1028    }
1029}
1030
1031/// The supervisor's view of one module's health, relayed to clients over
1032/// channel-0 and rendered by `ck health`.
1033///
1034/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1035/// stated here rather than only at the wire type a consumer reads. A reader can
1036/// look up what `None` means; only a writer can silently change it, and the
1037/// writer has no reason to go looking at a downstream contract before editing.
1038///
1039/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1040/// back to `None` on re-registration precisely so a respawned module does not
1041/// carry its predecessor's timestamp — so an old value and an absent one call for
1042/// opposite readings, and anything that defaulted this to a number would make a
1043/// never-probed module indistinguishable from one probed at the epoch.
1044///
1045/// `detail` and `metrics` are `None` when the module published none on this
1046/// probe, which does not mean it reported nothing wrong — it is also the shape
1047/// when the probe never reached it. `last_probe_ms` is what separates those.
1048#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050    pub status: SupervisorHealthStatus,
1051    pub last_probe_ms: Option<u64>,
1052    pub detail: Option<String>,
1053    pub metrics: Option<Value>,
1054    pub consecutive_failures: u32,
1055    /// Number of replies received after a recurring health probe's deadline.
1056    /// Unlike a timeout, every increment proves the module was alive.
1057    pub late_answer_count: u64,
1058    /// End-to-end latency of the newest late reply, measured from probe start.
1059    pub last_late_answer_latency_ms: Option<u64>,
1060    pub last_action: Option<String>,
1061    /// Set together with `last_action`; the pair moves as one, and both being
1062    /// absent means no escalation has ever been taken rather than that the last
1063    /// one succeeded.
1064    pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068    fn default() -> Self {
1069        Self {
1070            status: SupervisorHealthStatus::Unknown,
1071            last_probe_ms: None,
1072            detail: None,
1073            metrics: None,
1074            consecutive_failures: 0,
1075            late_answer_count: 0,
1076            last_late_answer_latency_ms: None,
1077            last_action: None,
1078            last_action_ms: None,
1079        }
1080    }
1081}
1082
1083/// Typed lifecycle state for a supervised module.
1084#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086    Starting,
1087    Running,
1088    Unresponsive,
1089    Restarting,
1090    Draining,
1091    Stopped,
1092    Failed,
1093    Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098        f.write_str(match self {
1099            Self::Starting => "starting",
1100            Self::Running => "running",
1101            Self::Unresponsive => "unresponsive",
1102            Self::Restarting => "restarting",
1103            Self::Draining => "draining",
1104            Self::Stopped => "stopped",
1105            Self::Failed => "failed",
1106            Self::Disabled => "disabled",
1107        })
1108    }
1109}
1110
1111/// Supervisor classification of a child-process exit.
1112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114    Clean,
1115    Crash,
1116    DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120    fn from(kind: ExitKind) -> Self {
1121        match kind {
1122            ExitKind::Clean => Self::Clean,
1123            ExitKind::Crash => Self::Crash,
1124            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125        }
1126    }
1127}
1128
1129/// Exact process identity retained when a supervised module registers its
1130/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133    pub(crate) pid: u32,
1134    pub(crate) start_time: u64,
1135}
1136
1137/// Last observed child exit, if any.
1138#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140    pub kind: ExitKind,
1141    pub code: Option<i32>,
1142    pub signal: Option<i32>,
1143    pub at_ms: u64,
1144}
1145
1146/// Point-in-time module status answerable by subc without forwarding to the
1147/// module process.
1148#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150    pub module_id: String,
1151    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1152    pub waiting_for_previous_pid: Option<u32>,
1153    /// Why the previous instance still prevents a new start.
1154    pub waiting_for_previous_detail: Option<String>,
1155    /// The daemon's Windows per-spawn record: `pipe` after the process it
1156    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1157    /// List/status reports use this record, not the source the module sends
1158    /// in its registration message's manifest.provenance.launch_nonce_source.
1159    pub launch_nonce_source: Option<String>,
1160    pub state: ModuleState,
1161    pub enabled: bool,
1162    pub process_alive: bool,
1163    pub registration_active: bool,
1164    /// The module's declared wire protocol, carried beside `live` because it is
1165    /// what makes `live` readable: the two fields answer one question together.
1166    /// While a process is alive this is its launch declaration, not a later
1167    /// pending-reload edit. When down it is the configured next launch protocol.
1168    pub protocol: ModuleProtocol,
1169    /// Whether the module is serving, under the strongest definition the daemon
1170    /// can assert for its protocol.
1171    ///
1172    /// A subc module must also be REGISTERED: its process being alive says
1173    /// nothing about whether it can take a request. A `protocol: "none"` module
1174    /// never registers, so that term is dropped and this falls back to "enabled,
1175    /// running, and the process the daemon launched is alive" -- which is all
1176    /// the daemon observes about a process that speaks no subc wire. It stays a
1177    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1178    /// rather than printing it bare.
1179    pub live: bool,
1180    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1181    /// restarts have already released their slot, so this count can go down
1182    /// without anybody touching the module.
1183    pub restart_count: u32,
1184    /// Replacement processes spawned over this module's entire supervisor lifetime;
1185    /// unlike `restart_count`, this value is never reset by an operator action
1186    /// and never falls out of a window.
1187    pub lifetime_restarts: u32,
1188    pub spawn_generation: u64,
1189    /// The budget `restart_count` is spent against. Carried alongside the count
1190    /// because the count alone does not say how close the module is to being
1191    /// disabled, and reporting one without the other is what makes an
1192    /// about-to-be-retired module look ordinary.
1193    pub max_restarts: u32,
1194    /// The span `restart_count` is counted over. Carried with the pair above for
1195    /// the same reason they are carried together: "2 of 3" means one thing for a
1196    /// ten-minute window and something else entirely for a lifetime.
1197    pub restart_window: Duration,
1198    /// Effective drain and restart timing policy used by this running module.
1199    /// These values are carried together with the restart budget so status
1200    /// readers can compare configured intent with what the supervisor applied.
1201    pub drain_timeout: Duration,
1202    pub restart_backoff: Duration,
1203    pub restart_max_backoff: Duration,
1204    /// The module's process. On macOS this stays absent while the `ck-subc`
1205    /// launch trampoline is still running in that pid, and appears once the
1206    /// exec acknowledgement confirms the module image has replaced it. Launch
1207    /// time and the supervisor's own hold on the process are unaffected.
1208    pub pid: Option<u32>,
1209    pub spawned_at_ms: Option<u64>,
1210    pub spawned_from: Option<PathBuf>,
1211    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212    pub process_start_time: Option<u64>,
1213    pub last_exit: Option<ExitReport>,
1214    pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220    generation: u64,
1221    turn: u64,
1222    registered_connection: Option<ConnectionId>,
1223    next_probe_at: Option<Instant>,
1224    wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230    #[cfg(windows)]
1231    pipe_consumed: Option<Arc<OnceLock<()>>>,
1232    /// Counts running-child loop turns, not executor polls of a parked wait.
1233    #[cfg(test)]
1234    actor_turns: u64,
1235    /// Running select after exec confirmation and with no unconsumed registry
1236    /// event. Recording a spawn or starting a loop turn is not this barrier.
1237    #[cfg(test)]
1238    actor_select: Option<ActorSelectCheckpoint>,
1239    state: ModuleState,
1240    enabled: bool,
1241    process_alive: bool,
1242    spawned_protocol: Option<ModuleProtocol>,
1243    spawn_failure: Option<String>,
1244    waiting_for_previous_pid: Option<u32>,
1245    waiting_for_previous_detail: Option<String>,
1246    /// When each crash restart was spent, oldest first. This IS the crash
1247    /// budget: its in-window length is the count an operator sees and the count
1248    /// the restart decision is made against, so there is no second counter that
1249    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1250    /// operator actions that used to zero the old lifetime counter.
1251    crash_restarts: VecDeque<Instant>,
1252    lifetime_restarts: u32,
1253    /// Successful child spawns in this daemon incarnation.
1254    ///
1255    /// `lifetime_restarts` was considered and rejected: it starts at zero
1256    /// (line 640), successful initial/operator spawns in `set_running` do not
1257    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1258    /// increments before a successful replacement exists (lines 604, 3846,
1259    /// and 3921), so a failed spawn can consume it. This counter moves only
1260    /// when a live PID is accepted below.
1261    spawn_generation: u64,
1262    pid: Option<u32>,
1263    #[cfg(target_os = "macos")]
1264    report_ready: Option<Arc<OnceLock<()>>>,
1265    /// Last reaped child, retained after current process facts are cleared.
1266    reaped_pid: Option<u32>,
1267    /// Whether a replacement has already been scheduled for the current process.
1268    respawn_pending: bool,
1269    /// A second restart is waiting for the replacement already scheduled.
1270    coalesced_restart_pending: bool,
1271    spawned_at_ms: Option<u64>,
1272    spawned_from: Option<PathBuf>,
1273    launch_spec_digest: Option<LaunchSpecDigest>,
1274    spawned_file_identity: Option<SpawnedFileIdentity>,
1275    process_start_time: Option<u64>,
1276    deliberate_severance: Option<ProcessIdentity>,
1277    last_exit: Option<ExitReport>,
1278    /// Diagnostic attached to the next drain's terminal record, if any.
1279    drain_disposition_detail: Option<String>,
1280    health: ModuleHealthStatus,
1281    /// Whether the current process was started as a swap candidate and so
1282    /// lives in the module's alternate cgroup. The next swap's candidate takes
1283    /// the other one, so the two processes of a swap never share a cgroup. A
1284    /// plain spawn always uses the primary cgroup.
1285    in_alternate_slot: bool,
1286    /// Whether the current `Draining` state ends in a replacement process
1287    /// (restart, reload, health restart) rather than a stop. Only meaningful
1288    /// while `state` is `Draining`; every entry into that state rewrites it.
1289    /// It is what lets route.open answer the retryable `module_reloading` to a
1290    /// consumer that reaches a still-registered process mid-restart, instead of
1291    /// the `supervisor_not_live` a stop or disable deserves.
1292    draining_to_replace: bool,
1293    /// Whether a configuration update has been applied since the current
1294    /// process was spawned, so that process runs an older spec than the one
1295    /// the supervisor now holds. A queued restart is only coalesced into a
1296    /// fresher process when this is false: a restart requested to pick up a
1297    /// new configuration must not be satisfied by a process that predates it.
1298    configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302    /// The pid that status, provenance and resource readings may report. While
1303    /// the launch trampoline still runs in the pid, reading its executable or
1304    /// resource use would describe `ck-subc`, not the module, so none is
1305    /// reported until the exec acknowledgement confirms the module image.
1306    fn reported_pid(&self) -> Option<u32> {
1307        #[cfg(target_os = "macos")]
1308        if self
1309            .report_ready
1310            .as_ref()
1311            .is_some_and(|ready| ready.get().is_none())
1312        {
1313            return None;
1314        }
1315        self.pid
1316    }
1317
1318    fn starting() -> Self {
1319        Self::new(ModuleState::Starting, true)
1320    }
1321
1322    fn disabled() -> Self {
1323        Self::new(ModuleState::Disabled, false)
1324    }
1325
1326    /// Crash restarts still inside `window`, having dropped the ones that are
1327    /// not. Pruning on read is what makes the budget a rate: an instant older
1328    /// than the window stops holding a slot the moment anybody counts.
1329    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330        while let Some(oldest) = self.crash_restarts.front() {
1331            if now.duration_since(*oldest) > window {
1332                self.crash_restarts.pop_front();
1333            } else {
1334                break;
1335            }
1336        }
1337        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338    }
1339
1340    /// Spend one unit of the crash budget and record the restart in the ledger.
1341    ///
1342    /// The ring is bounded by the cap because more than `max_restarts` in-window
1343    /// instants can never be reached (the caller refuses the restart first), so
1344    /// anything beyond that is an unbounded queue waiting to happen.
1345    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346        self.crash_restarts.push_back(now);
1347        while self.crash_restarts.len() > policy.max_restarts as usize {
1348            self.crash_restarts.pop_front();
1349        }
1350        self.lifetime_restarts += 1;
1351    }
1352
1353    /// Reserve one crash-restart slot and calculate the delay before respawning.
1354    /// The count is captured before recording this restart, so the first retry
1355    /// uses the base delay and each later in-window retry escalates once.
1356    fn next_crash_restart(
1357        &mut self,
1358        policy: &RestartPolicy,
1359        now: Instant,
1360    ) -> Option<CrashRestartSchedule> {
1361        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362        if restart_in_window >= policy.max_restarts {
1363            return None;
1364        }
1365        self.record_crash_restart(policy, now);
1366        Some(CrashRestartSchedule {
1367            restart_in_window,
1368            delay: policy.delay_for_restart(restart_in_window),
1369        })
1370    }
1371
1372    /// Give the module its full budget back, as an operator restart, reload, or
1373    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1374    /// ledger of what actually happened, and an operator action does not unmake
1375    /// the crashes.
1376    fn clear_crash_restarts(&mut self) {
1377        self.crash_restarts.clear();
1378    }
1379
1380    fn new(state: ModuleState, enabled: bool) -> Self {
1381        Self {
1382            exec_outcome: None,
1383            #[cfg(test)]
1384            actor_turns: 0,
1385            #[cfg(test)]
1386            actor_select: None,
1387            state,
1388            enabled,
1389            process_alive: false,
1390            #[cfg(windows)]
1391            pipe_consumed: None,
1392            spawned_protocol: None,
1393            spawn_failure: None,
1394            waiting_for_previous_pid: None,
1395            waiting_for_previous_detail: None,
1396            crash_restarts: VecDeque::new(),
1397            lifetime_restarts: 0,
1398            spawn_generation: 0,
1399            pid: None,
1400            #[cfg(target_os = "macos")]
1401            report_ready: None,
1402            reaped_pid: None,
1403            respawn_pending: false,
1404            coalesced_restart_pending: false,
1405            spawned_at_ms: None,
1406            spawned_from: None,
1407            launch_spec_digest: None,
1408            spawned_file_identity: None,
1409            process_start_time: None,
1410            deliberate_severance: None,
1411            last_exit: None,
1412            drain_disposition_detail: None,
1413            health: ModuleHealthStatus::default(),
1414            in_alternate_slot: false,
1415            draining_to_replace: false,
1416            configuration_updated_since_spawn: false,
1417        }
1418    }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427    version: u8,
1428    frames: mpsc::Sender<Frame>,
1429    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1430    /// from which event. The full frame channel cannot carry that news, so it
1431    /// travels beside it; see `SpawnEventFeed::subscribe`.
1432    lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437    daemon_incarnation: String,
1438    seq: u64,
1439    capacity: usize,
1440    live: HashMap<String, LiveSpawn>,
1441    generations: HashMap<String, u64>,
1442    events: VecDeque<SpawnEvent>,
1443    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447    fn default() -> Self {
1448        Self {
1449            daemon_incarnation: "unconfigured".to_string(),
1450            seq: 0,
1451            capacity: SPAWN_EVENT_RING_CAPACITY,
1452            live: HashMap::new(),
1453            generations: HashMap::new(),
1454            events: VecDeque::new(),
1455            subscribers: HashMap::new(),
1456        }
1457    }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462    Arc<Mutex<SpawnEventState>>,
1463    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468    ForeignIncarnation { current: String },
1469    TooOld { oldest: SpawnCursor },
1470    Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474    fn configure_incarnation(&self, daemon_incarnation: String) {
1475        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476        state.daemon_incarnation = daemon_incarnation;
1477        state.seq = 0;
1478        state.live.clear();
1479        state.generations.clear();
1480        state.events.clear();
1481        state.subscribers.clear();
1482    }
1483
1484    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485        SpawnCursor {
1486            daemon_incarnation: state.daemon_incarnation.clone(),
1487            seq: state.seq,
1488        }
1489    }
1490
1491    fn snapshot(&self) -> SpawnSnapshot {
1492        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495        SpawnSnapshot {
1496            cursor: Self::cursor(&state),
1497            ring_bound: state.capacity as u64,
1498            live,
1499        }
1500    }
1501
1502    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504        let generation = state
1505            .generations
1506            .get(module_id)
1507            .copied()
1508            .unwrap_or(0)
1509            .checked_add(1)
1510            .expect("spawn generation exhausted");
1511        state.generations.insert(module_id.to_string(), generation);
1512        let live = LiveSpawn {
1513            module_id: module_id.to_string(),
1514            spawn_generation: generation,
1515            pid,
1516            spawned_at_ms,
1517        };
1518        state.live.insert(module_id.to_string(), live);
1519        Self::emit_locked(
1520            &mut state,
1521            SpawnEventKind::Spawned,
1522            module_id.to_string(),
1523            generation,
1524            pid,
1525            None,
1526            None,
1527        );
1528        generation
1529    }
1530
1531    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533        let Some(live) = state.live.remove(module_id) else {
1534            warn!(
1535                module_id,
1536                "terminal record had no live spawn event identity"
1537            );
1538            return;
1539        };
1540        Self::emit_locked(
1541            &mut state,
1542            SpawnEventKind::Exited,
1543            module_id.to_string(),
1544            live.spawn_generation,
1545            live.pid,
1546            exit_code,
1547            exit_signal,
1548        );
1549    }
1550
1551    /// Report the exit of a process that a swap has already replaced.
1552    ///
1553    /// `emit_exited` removes the module's live entry, which after a swap's
1554    /// cutover describes the promoted candidate, not the old process now
1555    /// exiting. This emits the old generation's exit and leaves the live entry
1556    /// alone unless it still names that generation.
1557    fn emit_superseded_exited(
1558        &self,
1559        module_id: &str,
1560        spawn_generation: u64,
1561        pid: u32,
1562        exit_code: Option<i32>,
1563        exit_signal: Option<i32>,
1564    ) {
1565        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566        if state
1567            .live
1568            .get(module_id)
1569            .is_some_and(|live| live.spawn_generation == spawn_generation)
1570        {
1571            state.live.remove(module_id);
1572        }
1573        Self::emit_locked(
1574            &mut state,
1575            SpawnEventKind::Exited,
1576            module_id.to_string(),
1577            spawn_generation,
1578            pid,
1579            exit_code,
1580            exit_signal,
1581        );
1582    }
1583
1584    #[allow(clippy::too_many_arguments)]
1585    fn emit_locked(
1586        state: &mut SpawnEventState,
1587        kind: SpawnEventKind,
1588        module_id: String,
1589        spawn_generation: u64,
1590        pid: u32,
1591        exit_code: Option<i32>,
1592        exit_signal: Option<i32>,
1593    ) {
1594        state.seq = state
1595            .seq
1596            .checked_add(1)
1597            .expect("spawn event sequence exhausted");
1598        let event = SpawnEvent {
1599            cursor: Self::cursor(state),
1600            kind,
1601            module_id,
1602            spawn_generation,
1603            pid,
1604            exit_code,
1605            exit_signal,
1606        };
1607        state.events.push_back(event.clone());
1608        while state.events.len() > state.capacity {
1609            state.events.pop_front();
1610        }
1611        let body = match serde_json::to_vec(&event) {
1612            Ok(body) => body,
1613            Err(error) => {
1614                error!(%error, "failed to serialize supervisor spawn event");
1615                return;
1616            }
1617        };
1618        state.subscribers.retain(|(connection_id, corr), subscriber| {
1619            let frame = Frame::build_with_version(
1620                subscriber.version,
1621                FrameType::StreamData,
1622                control_flags(),
1623                0,
1624                0,
1625                *corr,
1626                body.clone(),
1627            );
1628            match frame {
1629                Ok(frame) => {
1630                    if subscriber.frames.try_send(frame).is_ok() {
1631                        true
1632                    } else {
1633                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634                        if let Some(lagged) = subscriber.lagged.take() {
1635                            let _ = lagged.send(event.cursor.clone());
1636                        }
1637                        false
1638                    }
1639                }
1640                Err(error) => {
1641                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642                    false
1643                }
1644            }
1645        });
1646    }
1647
1648    fn subscribe(
1649        &self,
1650        connection_id: ConnectionId,
1651        corr: u64,
1652        version: u8,
1653        since: Option<SpawnCursor>,
1654        sink: FrameSink,
1655    ) -> Result<(), SpawnSubscribeRefusal> {
1656        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658        {
1659            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660            let replay = if let Some(since) = since {
1661                if since.daemon_incarnation != state.daemon_incarnation {
1662                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663                        current: state.daemon_incarnation.clone(),
1664                    });
1665                }
1666                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667                    if since.seq < oldest.seq.saturating_sub(1) {
1668                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669                    }
1670                }
1671                state
1672                    .events
1673                    .iter()
1674                    .filter(|event| event.cursor.seq > since.seq)
1675                    .cloned()
1676                    .collect::<Vec<_>>()
1677            } else {
1678                Vec::new()
1679            };
1680            for event in replay {
1681                let body = serde_json::to_vec(&event)
1682                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683                let frame = Frame::build_with_version(
1684                    version,
1685                    FrameType::StreamData,
1686                    control_flags(),
1687                    0,
1688                    0,
1689                    corr,
1690                    body,
1691                )
1692                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693                frames
1694                    .try_send(frame)
1695                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696            }
1697            state.subscribers.insert(
1698                (connection_id, corr),
1699                SpawnSubscriber {
1700                    version,
1701                    frames,
1702                    lagged: Some(lagged),
1703                },
1704            );
1705        }
1706        // The lagged terminal is sent here, by the forwarder, rather than by
1707        // the emitter: at the moment of the drop the subscriber's own channel
1708        // is full, and writing to the connection sink directly from the emitter
1709        // would put the Error AHEAD of the events still queued in that channel
1710        // (and the emitter holds the feed lock, so it cannot await the sink).
1711        // Dropping the subscriber drops the only sender, so `recv` drains every
1712        // queued event and then returns `None`; only then is the Error sent, so
1713        // the client sees each event it can keep, then the reason it was cut.
1714        // Cancel and connection removal drop the oneshot unsent, so they end
1715        // the stream with no Error.
1716        tokio::spawn(async move {
1717            while let Some(frame) = receiver.recv().await {
1718                if sink.send(frame).await.is_err() {
1719                    return;
1720                }
1721            }
1722            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723                return;
1724            };
1725            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726                Ok(frame) => {
1727                    let _ = sink.send(frame).await;
1728                }
1729                Err(error) => {
1730                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731                }
1732            }
1733        });
1734        Ok(())
1735    }
1736
1737    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738        let Some(subscriber) = self
1739            .0
1740            .lock()
1741            .unwrap_or_else(|p| p.into_inner())
1742            .subscribers
1743            .remove(&(connection_id, corr))
1744        else {
1745            return false;
1746        };
1747        if let Ok(frame) = Frame::build_with_version(
1748            subscriber.version,
1749            FrameType::StreamEnd,
1750            control_flags(),
1751            0,
1752            0,
1753            corr,
1754            Vec::new(),
1755        ) {
1756            tokio::spawn(async move {
1757                let _ = subscriber.frames.send(frame).await;
1758            });
1759        }
1760        true
1761    }
1762
1763    fn remove_connection(&self, connection_id: ConnectionId) {
1764        self.0
1765            .lock()
1766            .unwrap_or_else(|p| p.into_inner())
1767            .subscribers
1768            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769    }
1770
1771    #[cfg(any(test, feature = "test-support"))]
1772    fn set_capacity(&self, capacity: usize) {
1773        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774    }
1775
1776    #[cfg(any(test, feature = "test-support"))]
1777    fn subscriber_count(&self) -> usize {
1778        self.0
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .subscribers
1782            .len()
1783    }
1784}
1785
1786/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1787/// The terminal Error a lagged spawn subscriber receives after its queued events.
1788fn spawn_subscriber_lagged_frame(
1789    version: u8,
1790    corr: u64,
1791    first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796            .to_string(),
1797        detail: Some(serde_json::json!({
1798            "first_undelivered_cursor": first_undelivered
1799        })),
1800    })
1801    .map_err(|error| error.to_string())?;
1802    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803        .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807    fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809    /// Whether the supervisor is replacing this module's process right now: an
1810    /// operator restart or reload, a health restart, or a crash respawn whose
1811    /// backoff is running. A module in that state is not live, but a consumer
1812    /// refused now should retry shortly rather than treat the target as gone.
1813    /// Stopped, failed, and disabled modules are not replacing.
1814    fn process_replacing(&self, _module_id: &str) -> bool {
1815        false
1816    }
1817}
1818
1819/// Shared process-liveness registry keyed by supervised `module_id`.
1820#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826    pub fn new() -> Self {
1827        Self::default()
1828    }
1829
1830    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831        let mut snapshots = self
1832            .snapshots
1833            .lock()
1834            .unwrap_or_else(|poisoned| poisoned.into_inner());
1835        snapshots.insert(module_id, snapshot);
1836    }
1837
1838    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        let is_current = snapshots
1844            .get(module_id)
1845            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846            .unwrap_or(false);
1847        if is_current {
1848            snapshots.remove(module_id);
1849        }
1850    }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854    fn process_live(&self, module_id: &str) -> Option<bool> {
1855        let snapshot = {
1856            let snapshots = self
1857                .snapshots
1858                .lock()
1859                .unwrap_or_else(|poisoned| poisoned.into_inner());
1860            snapshots.get(module_id).cloned()
1861        }?;
1862        let snapshot = snapshot
1863            .lock()
1864            .unwrap_or_else(|poisoned| poisoned.into_inner());
1865        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866    }
1867
1868    fn process_replacing(&self, module_id: &str) -> bool {
1869        let Some(snapshot) = self
1870            .snapshots
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner())
1873            .get(module_id)
1874            .cloned()
1875        else {
1876            return false;
1877        };
1878        let snapshot = snapshot
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner());
1881        snapshot.enabled
1882            && match snapshot.state {
1883                ModuleState::Restarting => true,
1884                ModuleState::Draining => snapshot.draining_to_replace,
1885                ModuleState::Starting
1886                | ModuleState::Running
1887                | ModuleState::Unresponsive
1888                | ModuleState::Stopped
1889                | ModuleState::Failed
1890                | ModuleState::Disabled => false,
1891            }
1892    }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898    reached: tokio::sync::Notify,
1899    resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904    Spawn,
1905    Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910    deadline: Instant,
1911    kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918    start_requester: Option<Principal>,
1919    pending_activity_restart: bool,
1920    provenance_probe: ExecutableIdentityProbe,
1921    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1922    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923    /// A reload acknowledges completion only after its replacement registers.
1924    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925    restart_policy: RestartPolicy,
1926    /// This module's RESOLVED drain budget: per-module config when present,
1927    /// else `default_drain_timeout`.
1928    drain_timeout: Duration,
1929    /// Shared with the status handle so the attested value changes atomically
1930    /// when a rescan updates the running drain policy.
1931    effective_drain_timeout: Arc<Mutex<Duration>>,
1932    /// The supervisor-wide fallback, kept so a configuration update that
1933    /// REMOVES the per-module override can re-resolve to it.
1934    default_drain_timeout: Duration,
1935    health: HealthConfig,
1936    connection_file_path: Option<PathBuf>,
1937    capture_logs_dir: Option<PathBuf>,
1938    forwarding: Option<Arc<ForwardingTable>>,
1939    /// The shared handle, so every spawn path (initial, restart, reload) records the
1940    /// reserved-module launch nonce the HELLO verifier checks against.
1941    supervisor_handle: Option<SupervisorHandle>,
1942    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1943    /// status queries.
1944    ///
1945    /// One ring per module, held across every respawn. The lines explaining an exit
1946    /// are written BEFORE that exit, so a ring recreated per process would be empty
1947    /// exactly when it is asked for.
1948    stderr_ring: Arc<Mutex<StderrRing>>,
1949    terminal_ring: Arc<Mutex<TerminalRing>>,
1950    spawn_events: SpawnEventFeed,
1951    child_roster: ChildRoster,
1952    #[cfg(target_os = "linux")]
1953    cgroup_placement: Option<subc_cgroup::Placement>,
1954    #[cfg(test)]
1955    test_seed_stale_facts_before_enable_spawn: bool,
1956    #[cfg(test)]
1957    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962    spec: ModuleSpec,
1963    health: HealthConfig,
1964}
1965
1966/// Shared daemon lookup table for supervised module handles.
1967///
1968/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1969/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1970/// launch nonces recorded at spawn are checked by the same daemon instance.
1971#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976    /// Module ids the supervisor has taken on. An id is added BEFORE the
1977    /// module's first process is spawned and removed only when the module
1978    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1979    /// the keys of `modules`.
1980    ///
1981    /// `modules` cannot answer "is this module configured?" on its own: a
1982    /// [`SupervisedModule`] only exists once its process has been spawned, and
1983    /// a fast child can connect, register, sync its scopes and ask about them
1984    /// before the supervisor has inserted it. Answering "not configured" in that
1985    /// gap makes scope admission refuse with the terminal "will never sync"
1986    /// instead of the retryable "has not synced yet".
1987    configured_ids: Arc<Mutex<HashSet<String>>>,
1988    spawn_events: SpawnEventFeed,
1989    /// The current expected launch nonce for each reserved module_id. Set when the
1990    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1991    /// non-reserved module never has an entry here and is never nonce-checked.
1992    /// Reserved module ids and the nonce that authorizes their next HELLO.
1993    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1994    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1995    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1996    /// had NO entry and admitted anyone: the reservation protected the nonce
1997    /// holder, not the NAME (found live by CKCRED's canary probe registering
1998    /// against a reserved scratch id).
1999    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2001    ///
2002    /// This is deliberately in-memory only: subc is state-free across daemon
2003    /// restarts, and the tombstone only explains the hours-after-removal window
2004    /// while this executing daemon is still alive. Do not persist it in a store.
2005    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006    /// The current launch nonce for every supervised spawn. This is separate from
2007    /// reserved_nonces because consumer route.open attestation applies to all spawned
2008    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2009    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010    /// Reserved namespace prefixes mapped to the supervised owner module whose
2011    /// current spawn nonce authorizes HELLO claims below the prefix.
2012    ///
2013    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2014    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2015    /// accidental collisions and lower-trust processes from squatting protected
2016    /// namespaces.
2017    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018    /// Blue/green swaps in progress, by module id. An entry exists from just
2019    /// before the candidate process is spawned until the swap has failed, or
2020    /// has cut over and the old process is gone. While it exists, HELLO for the
2021    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2022    /// consumer attestation accepts both processes' nonces.
2023    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2025    promotion_observer: PromotionObserverSlot,
2026    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2027    /// this daemon-wide ordering, a rescan could retire or update a module while a
2028    /// concurrent reload still held its old handle and launch specification.
2029    operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032/// Told when a swap has promoted its candidate to be the module's active
2033/// registration.
2034///
2035/// An ordinary HELLO runs the control plane's registration side effects (the
2036/// capability cache, the deny census, the requirement recompute) as it
2037/// registers. A swap candidate's HELLO does not, because it is not routable;
2038/// promotion is when those must run instead, and promotion happens in the
2039/// supervisor, which has no other way into the control handler.
2040pub(crate) trait SwapPromotionObserver: Send + Sync {
2041    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2045/// control handler) owns this handle, so a strong reference back would be a
2046/// cycle that keeps both alive.
2047#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052        f.write_str("PromotionObserverSlot")
2053    }
2054}
2055
2056/// The nonces of one open swap.
2057#[derive(Debug, Clone)]
2058struct OpenSwap {
2059    /// The launch nonce minted for the candidate process. It is the swap
2060    /// token: the only thing that admits a HELLO into the candidate slot.
2061    candidate_nonce: String,
2062    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2063    /// here because cutover moves the module's recorded spawn nonce to the
2064    /// candidate while the incumbent is still draining and its consumers are
2065    /// still attesting with this one.
2066    incumbent_nonce: Option<String>,
2067    /// Set once a HELLO has been admitted with the swap token, so the token
2068    /// admits one registration and cannot be replayed after cutover empties
2069    /// the candidate slot.
2070    candidate_admitted: bool,
2071}
2072
2073/// What the swap gate says about a HELLO. See
2074/// [`SupervisorHandle::swap_hello_admission`].
2075#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077    /// No swap is open for the id (or the HELLO carries the incumbent's own
2078    /// nonce); the ordinary gates decide.
2079    NotSwapping,
2080    /// The HELLO carries the swap token: register it into the candidate slot.
2081    Candidate,
2082    /// A swap is open and the HELLO carries a nonce the supervisor did not
2083    /// mint for this id, no nonce, or a token already used.
2084    Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089    Exact {
2090        module_id: String,
2091    },
2092    Prefix {
2093        prefix: String,
2094        owner_module_id: String,
2095    },
2096}
2097
2098impl SupervisorHandle {
2099    pub fn new() -> Self {
2100        Self::default()
2101    }
2102
2103    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104        self.spawn_events.snapshot()
2105    }
2106
2107    pub(crate) fn subscribe_spawns(
2108        &self,
2109        connection_id: ConnectionId,
2110        corr: u64,
2111        version: u8,
2112        since: Option<SpawnCursor>,
2113        sink: FrameSink,
2114    ) -> Result<(), SpawnSubscribeRefusal> {
2115        self.spawn_events
2116            .subscribe(connection_id, corr, version, since, sink)
2117    }
2118
2119    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120        self.spawn_events.cancel(connection_id, corr)
2121    }
2122
2123    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124        self.spawn_events.remove_connection(connection_id);
2125    }
2126
2127    #[cfg(any(test, feature = "test-support"))]
2128    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129        assert!(capacity > 0, "spawn event capacity must be non-zero");
2130        self.spawn_events.set_capacity(capacity);
2131    }
2132
2133    #[cfg(any(test, feature = "test-support"))]
2134    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135        self.spawn_events.subscriber_count()
2136    }
2137
2138    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2139    /// a respawn invalidates stale consumer identities.
2140    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141        self.spawn_nonces
2142            .lock()
2143            .unwrap_or_else(|poisoned| poisoned.into_inner())
2144            .insert(module_id.to_string(), nonce);
2145    }
2146
2147    /// Record the launch nonce expected from the next HELLO for a reserved module,
2148    /// replacing any prior nonce (a respawn invalidates the previous one).
2149    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150        self.reserved_nonces
2151            .lock()
2152            .unwrap_or_else(|poisoned| poisoned.into_inner())
2153            .insert(module_id.to_string(), Some(nonce));
2154    }
2155
2156    /// Record namespace prefixes owned by a supervised module.
2157    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158        let mut owners = self
2159            .reserved_prefix_owners
2160            .lock()
2161            .unwrap_or_else(|poisoned| poisoned.into_inner());
2162        owners.retain(|_, owner| owner != owner_module_id);
2163        for prefix in prefixes {
2164            owners.insert(prefix.clone(), owner_module_id.to_string());
2165        }
2166    }
2167
2168    /// The launch nonce most recently minted for a module's spawn, if any.
2169    #[cfg(test)]
2170    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171        self.spawn_nonces
2172            .lock()
2173            .unwrap_or_else(|poisoned| poisoned.into_inner())
2174            .get(module_id)
2175            .cloned()
2176    }
2177
2178    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180        let spawn_nonce = self
2181            .spawn_nonces
2182            .lock()
2183            .unwrap_or_else(|poisoned| poisoned.into_inner())
2184            .get(&spec.module_id)
2185            .cloned();
2186        let mut reserved_nonces = self
2187            .reserved_nonces
2188            .lock()
2189            .unwrap_or_else(|poisoned| poisoned.into_inner());
2190        if spec.reserved {
2191            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2192            // reserved name whose module has never spawned has no legitimate
2193            // holder, and the entry's absence is what used to leave the name
2194            // open to the first claimant.
2195            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196        }
2197        drop(reserved_nonces);
2198        // A later unreserved declaration must not silently unreserve an id that
2199        // was retained after its reserved configuration was removed. The explicit
2200        // release ceremony is the only operation that retires that gate.
2201        self.removal_tombstones
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .remove(&spec.module_id);
2205    }
2206
2207    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2208    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2209    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2210    /// with no matching prefix are always authorized.
2211    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212        self.reserved_hello_rejection(module_id, presented)
2213            .is_none()
2214    }
2215
2216    pub(crate) fn reserved_hello_rejection(
2217        &self,
2218        module_id: &str,
2219        presented: Option<&str>,
2220    ) -> Option<ReservedHelloRejection> {
2221        let nonces = self
2222            .reserved_nonces
2223            .lock()
2224            .unwrap_or_else(|poisoned| poisoned.into_inner());
2225        if let Some(expected) = nonces.get(module_id) {
2226            // `None` = reserved with no legitimate holder: refuse every
2227            // presentation, because no process can hold a nonce that was never
2228            // minted. Only a real minted nonce admits, in constant time.
2229            let authorized = match expected {
2230                Some(expected) => {
2231                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232                }
2233                None => false,
2234            };
2235            if authorized {
2236                return None;
2237            }
2238            return Some(ReservedHelloRejection::Exact {
2239                module_id: module_id.to_string(),
2240            });
2241        }
2242        drop(nonces);
2243
2244        let matched_prefix = self
2245            .reserved_prefix_owners
2246            .lock()
2247            .unwrap_or_else(|poisoned| poisoned.into_inner())
2248            .iter()
2249            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250            .max_by_key(|(prefix, _)| prefix.len())
2251            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252        let (prefix, owner_module_id) = matched_prefix?;
2253
2254        let authorized = presented.is_some_and(|presented| {
2255            self.spawn_nonces
2256                .lock()
2257                .unwrap_or_else(|poisoned| poisoned.into_inner())
2258                .get(&owner_module_id)
2259                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260                // While the owner is being swapped, children started by
2261                // either of its two processes hold that process's nonce.
2262                || self.swap_nonce_matches(&owner_module_id, presented)
2263        });
2264        if authorized {
2265            None
2266        } else {
2267            Some(ReservedHelloRejection::Prefix {
2268                prefix,
2269                owner_module_id,
2270            })
2271        }
2272    }
2273
2274    /// Whether a consumer connection proved it came from a daemon-spawned module.
2275    ///
2276    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2277    /// accepted only for module ids the supervisor has spawned.
2278    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279        if presented.is_empty() {
2280            return false;
2281        }
2282        let nonces = self
2283            .spawn_nonces
2284            .lock()
2285            .unwrap_or_else(|poisoned| poisoned.into_inner());
2286        let current = nonces
2287            .get(module_id)
2288            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289        drop(nonces);
2290        // During a swap two processes of the module are alive, and a consumer
2291        // started by either one presents that process's nonce. Accepting only
2292        // the recorded one would fail the incumbent's consumers for the whole
2293        // overlap once cutover moves the record to the candidate.
2294        current || self.swap_nonce_matches(module_id, presented)
2295    }
2296
2297    /// Whether `presented` is either nonce of an open swap for `module_id`.
2298    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299        let swaps = self
2300            .swaps
2301            .lock()
2302            .unwrap_or_else(|poisoned| poisoned.into_inner());
2303        swaps.get(module_id).is_some_and(|swap| {
2304            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307                })
2308        })
2309    }
2310
2311    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2312    /// Called before the candidate process exists.
2313    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314        let incumbent_nonce = self
2315            .spawn_nonces
2316            .lock()
2317            .unwrap_or_else(|poisoned| poisoned.into_inner())
2318            .get(module_id)
2319            .cloned();
2320        self.swaps
2321            .lock()
2322            .unwrap_or_else(|poisoned| poisoned.into_inner())
2323            .insert(
2324                module_id.to_string(),
2325                OpenSwap {
2326                    candidate_nonce,
2327                    incumbent_nonce,
2328                    candidate_admitted: false,
2329                },
2330            );
2331    }
2332
2333    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2334    /// the module's recorded one.
2335    pub(crate) fn close_swap(&self, module_id: &str) {
2336        self.swaps
2337            .lock()
2338            .unwrap_or_else(|poisoned| poisoned.into_inner())
2339            .remove(module_id);
2340    }
2341
2342    /// Install the observer told about swap promotions, replacing any earlier
2343    /// one.
2344    pub(crate) fn set_swap_promotion_observer(
2345        &self,
2346        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347    ) {
2348        *self
2349            .promotion_observer
2350            .0
2351            .lock()
2352            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353    }
2354
2355    /// Tell the installed observer, if it is still alive, that a swap promoted
2356    /// `registration`.
2357    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358        let observer = self
2359            .promotion_observer
2360            .0
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner())
2363            .as_ref()
2364            .and_then(std::sync::Weak::upgrade);
2365        if let Some(observer) = observer {
2366            observer.swap_promoted(registration);
2367        }
2368    }
2369
2370    /// Whether a swap is open for `module_id`.
2371    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372        self.swaps
2373            .lock()
2374            .unwrap_or_else(|poisoned| poisoned.into_inner())
2375            .contains_key(module_id)
2376    }
2377
2378    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2379    /// respawn would, once cutover has made the candidate the module's process.
2380    /// The swap stays open so the incumbent's nonce keeps attesting until the
2381    /// incumbent has drained and exited.
2382    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383        let candidate_nonce = self
2384            .swaps
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .get(module_id)
2388            .map(|swap| swap.candidate_nonce.clone());
2389        let Some(nonce) = candidate_nonce else {
2390            return;
2391        };
2392        self.set_spawn_nonce(module_id, nonce.clone());
2393        if reserved {
2394            self.set_reserved_nonce(module_id, nonce);
2395        }
2396    }
2397
2398    /// The swap gate for a HELLO claiming `module_id`.
2399    ///
2400    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2401    /// presents the candidate nonce, which the reserved gate (holding the
2402    /// incumbent's nonce) would refuse as `reserved_module` before swap
2403    /// admission was ever reached. And it applies to unreserved ids too: for an
2404    /// unreserved id the only thing that ever stopped a second process claiming
2405    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2406    /// refusal a swap lifts for its candidate.
2407    ///
2408    /// The incumbent's own nonce falls through to the ordinary gates, which
2409    /// treat it as they always have (a live incumbent is refused as a
2410    /// duplicate). Anything else while a swap is open is refused, including an
2411    /// absent nonce.
2412    pub(crate) fn swap_hello_admission(
2413        &self,
2414        module_id: &str,
2415        presented: Option<&str>,
2416    ) -> SwapHelloAdmission {
2417        let swaps = self
2418            .swaps
2419            .lock()
2420            .unwrap_or_else(|poisoned| poisoned.into_inner());
2421        let Some(swap) = swaps.get(module_id) else {
2422            return SwapHelloAdmission::NotSwapping;
2423        };
2424        let Some(presented) = presented else {
2425            return SwapHelloAdmission::Refused;
2426        };
2427        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428            return if swap.candidate_admitted {
2429                SwapHelloAdmission::Refused
2430            } else {
2431                SwapHelloAdmission::Candidate
2432            };
2433        }
2434        if swap
2435            .incumbent_nonce
2436            .as_deref()
2437            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438        {
2439            return SwapHelloAdmission::NotSwapping;
2440        }
2441        SwapHelloAdmission::Refused
2442    }
2443
2444    /// Record that the swap token has registered a candidate, so it admits no
2445    /// second HELLO.
2446    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447        if let Some(swap) = self
2448            .swaps
2449            .lock()
2450            .unwrap_or_else(|poisoned| poisoned.into_inner())
2451            .get_mut(module_id)
2452        {
2453            swap.candidate_admitted = true;
2454        }
2455    }
2456
2457    /// Test/support lookup for the current launch nonce of a supervised spawn.
2458    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459        self.spawn_nonces
2460            .lock()
2461            .unwrap_or_else(|poisoned| poisoned.into_inner())
2462            .get(module_id)
2463            .cloned()
2464    }
2465
2466    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2467    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468        self.reserved_nonces
2469            .lock()
2470            .unwrap_or_else(|poisoned| poisoned.into_inner())
2471            .get(module_id)
2472            .cloned()
2473            .flatten()
2474    }
2475
2476    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477        // Normally already marked before the process was spawned; marking here
2478        // too keeps `configured_ids` a superset of the roster for any caller
2479        // that inserts a module directly.
2480        self.mark_configured(module.module_id());
2481        let mut modules = self
2482            .modules
2483            .lock()
2484            .unwrap_or_else(|poisoned| poisoned.into_inner());
2485        modules.insert(module.module_id().to_string(), module)
2486    }
2487
2488    /// Record that the supervisor has taken on `module_id`. Called before the
2489    /// module's first process is spawned, so that by the time that process can
2490    /// register, [`Self::is_configured`] already answers true.
2491    fn mark_configured(&self, module_id: &str) {
2492        self.configured_ids
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner())
2495            .insert(module_id.to_string());
2496    }
2497
2498    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2499    /// before it was ever put on the roster. A module already on the roster
2500    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2501    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502        let modules = self
2503            .modules
2504            .lock()
2505            .unwrap_or_else(|poisoned| poisoned.into_inner());
2506        if !modules.contains_key(module_id) {
2507            self.configured_ids
2508                .lock()
2509                .unwrap_or_else(|poisoned| poisoned.into_inner())
2510                .remove(module_id);
2511        }
2512    }
2513
2514    /// Whether `module_id` is a module this daemon supervises: on the roster,
2515    /// or about to be (its process is being spawned right now).
2516    ///
2517    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2518    /// for scopes: a supervised module's process can register and sync before
2519    /// [`Self::get`] can return it, and in that window it is still a module
2520    /// that will sync, not one that never will.
2521    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522        self.configured_ids
2523            .lock()
2524            .unwrap_or_else(|poisoned| poisoned.into_inner())
2525            .contains(module_id)
2526    }
2527
2528    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529        let modules = self
2530            .modules
2531            .lock()
2532            .unwrap_or_else(|poisoned| poisoned.into_inner());
2533        modules.get(module_id).cloned()
2534    }
2535
2536    pub(crate) fn record_late_health_answer(
2537        &self,
2538        module_id: &str,
2539        latency_ms: u64,
2540    ) -> Result<bool, SuperviseError> {
2541        let Some(module) = self.get(module_id) else {
2542            return Ok(false);
2543        };
2544        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546            state.health.last_late_answer_latency_ms = Some(latency_ms);
2547            // A late answer is an answer: the module served the probe, just past
2548            // the deadline. Leaving the miss streak in place while logging
2549            // "proves the module is alive" is how a CPU-starved module that
2550            // answers every probe a few seconds late still marches to the
2551            // threshold and gets killed — the exact kill class `NoAnswer` is
2552            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2553            // is degradation, and degradation reports; it does not restart.
2554            state.health.consecutive_failures = 0;
2555        })?;
2556        Ok(true)
2557    }
2558
2559    /// Arm the one-shot marker for the module process that this caller
2560    /// deliberately initiated severance against. Generic connection teardown
2561    /// must not call this:
2562    /// a surviving process would otherwise retain an exemption for a later
2563    /// genuine crash.
2564    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565        let Some(module) = self.get(module_id) else {
2566            return Ok(false);
2567        };
2568        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570            return Ok(false);
2571        };
2572        drop(snapshot);
2573        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574    }
2575
2576    pub fn list(&self) -> Vec<SupervisedModule> {
2577        let modules = self
2578            .modules
2579            .lock()
2580            .unwrap_or_else(|poisoned| poisoned.into_inner());
2581        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583        modules
2584    }
2585
2586    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587        self.activity_images
2588            .lock()
2589            .unwrap_or_else(|p| p.into_inner())
2590            .remove(module_id);
2591        self.spawn_nonces
2592            .lock()
2593            .unwrap_or_else(|poisoned| poisoned.into_inner())
2594            .remove(module_id);
2595        self.close_swap(module_id);
2596        let mut reserved_nonces = self
2597            .reserved_nonces
2598            .lock()
2599            .unwrap_or_else(|poisoned| poisoned.into_inner());
2600        if reserved_nonces.contains_key(module_id) {
2601            // The old nonce must die with the removed process, but the exact-id
2602            // gate remains until an operator explicitly releases it.
2603            reserved_nonces.insert(module_id.to_string(), None);
2604        }
2605        drop(reserved_nonces);
2606        self.reserved_prefix_owners
2607            .lock()
2608            .unwrap_or_else(|poisoned| poisoned.into_inner())
2609            .retain(|_, owner| owner != module_id);
2610        let removed = self
2611            .modules
2612            .lock()
2613            .unwrap_or_else(|poisoned| poisoned.into_inner())
2614            .remove(module_id);
2615        self.configured_ids
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        removed
2620    }
2621
2622    /// Remember a module removed by a non-preview rescan so route.open can
2623    /// distinguish that intentional removal from an unknown id.
2624    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625        self.removal_tombstones
2626            .lock()
2627            .unwrap_or_else(|poisoned| poisoned.into_inner())
2628            .insert(module_id.to_string(), unix_ms_now());
2629    }
2630
2631    /// Return how long ago a rescan removed this module in milliseconds.
2632    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633        self.removal_tombstones
2634            .lock()
2635            .unwrap_or_else(|poisoned| poisoned.into_inner())
2636            .get(module_id)
2637            .copied()
2638            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639    }
2640
2641    /// Retire a reserved-id gate only after its module has left supervision.
2642    ///
2643    /// A retained gate has no live nonce (`None`), so releasing any other entry
2644    /// would weaken a currently configured or otherwise active reservation.
2645    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646        if self.get(module_id).is_some() {
2647            return false;
2648        }
2649        let mut reserved_nonces = self
2650            .reserved_nonces
2651            .lock()
2652            .unwrap_or_else(|poisoned| poisoned.into_inner());
2653        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654            return false;
2655        }
2656        reserved_nonces.remove(module_id);
2657        true
2658    }
2659
2660    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661        Arc::clone(&self.operation_lock)
2662    }
2663}
2664
2665/// Process supervisor for subc-owned singleton modules.
2666#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668    registry: Arc<Registry>,
2669    restart_policy: RestartPolicy,
2670    drain_timeout: Duration,
2671    connection_file_path: Option<PathBuf>,
2672    capture_logs_dir: Option<PathBuf>,
2673    forwarding: Option<Arc<ForwardingTable>>,
2674    process_liveness: Arc<SupervisorProcessLiveness>,
2675    supervisor_handle: Option<SupervisorHandle>,
2676    health: HealthConfig,
2677    daemon_start_clock: crate::clock::StartClock,
2678    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679    spawn_events: SpawnEventFeed,
2680    provenance_probe: ExecutableIdentityProbe,
2681    /// Every process spawned through this supervisor (and its clones) and not
2682    /// yet reaped, so daemon shutdown can end them.
2683    child_roster: ChildRoster,
2684    #[cfg(target_os = "linux")]
2685    cgroup_placement: Option<subc_cgroup::Placement>,
2686    #[cfg(test)]
2687    test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690/// A roster entry whose monitor and first process have not started yet.
2691/// Keeping the command receiver here lets discovery serve status immediately
2692/// while start/stop requests wait until bootstrap has published the connection file.
2693pub(crate) struct RegisteredConfiguredModule {
2694    spec: ModuleSpec,
2695    runtime: SupervisorRuntimeConfig,
2696    module: SupervisedModule,
2697    commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700/// Test-only hook run on the path that takes on a new module, right after its
2701/// first `spawn_child` returns (the process exists and could already be
2702/// registering) and before that process is handed to the module's supervise
2703/// loop and put on the roster. Lets a test observe what a fast child would see
2704/// in that window without racing a real one.
2705#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715        f.write_str("AfterFirstSpawnHook")
2716    }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721    fn run(&self, module_id: &str) {
2722        if let Some(hook) = &self.0 {
2723            hook(module_id);
2724        }
2725    }
2726}
2727
2728impl Supervisor {
2729    #[cfg(test)]
2730    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731        let supervisor = Self::new(registry, policy);
2732        #[cfg(target_os = "macos")]
2733        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734        supervisor
2735    }
2736    /// Verify the trampoline once when configured. Missing private OS support
2737    /// refuses every macOS launch by name but does not stop the daemon's control
2738    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2739    /// entry point; the library must not exec an arbitrary hosting program.
2740    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741        let path = path.into();
2742        #[cfg(target_os = "macos")]
2743        {
2744            let result = probe_privacy_trampoline(&path).map(|()| path);
2745            if let Err(cause) = &result {
2746                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747            }
2748            self.child_roster.set_privacy_trampoline(result);
2749        }
2750        #[cfg(not(target_os = "macos"))]
2751        let _ = path;
2752        self
2753    }
2754    /// The first step of an announced daemon shutdown, before the notice and
2755    /// before any connection is closed.
2756    ///
2757    /// Sets the daemon-shutdown flag first: from here on no module is
2758    /// respawned (crash restart, operator restart, or swap), and every child
2759    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2760    /// the module exits on the EOF this shutdown gives it or is signalled by a
2761    /// service manager that kills the whole cgroup. Then writes the journal's
2762    /// shutdown marker, which records the instant and closes this daemon
2763    /// incarnation's stretch of the journal.
2764    #[cfg(unix)]
2765    pub(crate) fn begin_daemon_shutdown(&self) {
2766        self.child_roster.close();
2767        if let Some(journal) = &self.terminal_journal {
2768            journal.stamp_shutdown();
2769        }
2770    }
2771
2772    /// Announce a cut while established connections can still carry replies.
2773    /// These budgets promise notice and a bounded wait, not child completion;
2774    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2775    #[cfg(unix)]
2776    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2777        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2778        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2779        let Some(forwarding) = &self.forwarding else {
2780            return Ok(());
2781        };
2782        let modules = forwarding
2783            .begin_daemon_drain()
2784            .map_err(SuperviseError::Forwarding)?;
2785        let deadline_ms =
2786            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2787        let mut notices = tokio::task::JoinSet::new();
2788        let mut drains = Vec::new();
2789        for (module_id, endpoint) in modules {
2790            let Some(target) = forwarding
2791                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2792                .map_err(SuperviseError::Forwarding)?
2793            else {
2794                continue;
2795            };
2796            let routes = forwarding
2797                .endpoint_routes(target.endpoint)
2798                .map_err(SuperviseError::Forwarding)?;
2799            // Restart allows deployed consumers to reopen after the new daemon
2800            // appears. The wire reason stays `restart`; what tells a daemon cut
2801            // apart from a module restart afterwards is the terminal record
2802            // itself, whose disposition is `daemon_shutdown` for every exit
2803            // observed once `begin_daemon_shutdown` has run.
2804            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2805                reason: RouteCloseReason::Restart,
2806                deadline_ms,
2807            })
2808            .expect("module draining serializes");
2809            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2810            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2811            for route in routes {
2812                let client = route.goodbye_target;
2813                if let Some((_, channels)) = clients
2814                    .iter_mut()
2815                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2816                {
2817                    channels.push(client.channel);
2818                } else {
2819                    let channel = client.channel;
2820                    clients.push((client, vec![channel]));
2821                }
2822            }
2823            for (client, mut channels) in clients {
2824                channels.sort_unstable();
2825                channels.dedup();
2826                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2827                    module_id: module_id.clone(),
2828                    channels,
2829                    reason: RouteCloseReason::Restart,
2830                })
2831                .expect("route closing serializes");
2832                recipients.push((client.sink, client.negotiated_ver, closing));
2833            }
2834            for (sink, version, body) in recipients {
2835                notices.spawn(async move {
2836                    let frame = Frame::build_with_version(
2837                        version,
2838                        FrameType::Push,
2839                        control_flags(),
2840                        0,
2841                        0,
2842                        0,
2843                        body,
2844                    )
2845                    .expect("bounded lifecycle notice frame builds");
2846                    sink.send_flushed(frame).await
2847                });
2848            }
2849            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2850            drains.push((module_id, target.endpoint, gauges));
2851        }
2852        // A quiet forwarding table is not proof that queued notices reached the
2853        // socket. Wait for writer flush acknowledgements before testing quiescence.
2854        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2855        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2856            if !matches!(result, Ok(Ok(()))) {
2857                warn!(?result, "daemon shutdown notice delivery failed");
2858            }
2859        }
2860        notices.abort_all();
2861        let deadline = Instant::now() + DRAIN_BUDGET;
2862        let mut waits = tokio::task::JoinSet::new();
2863        for (module_id, endpoint, gauges) in drains {
2864            let forwarding = Arc::clone(forwarding);
2865            let mut runtime = self.runtime_config();
2866            runtime.health.cadence = Duration::from_millis(100);
2867            waits.spawn(async move {
2868                let wait_result = wait_for_forwarding_quiescence(
2869                    &forwarding,
2870                    &module_id,
2871                    &runtime,
2872                    endpoint,
2873                    deadline,
2874                    &gauges,
2875                    DrainScope::Endpoint(endpoint),
2876                    &mut DrainGaugeTrace::new(&gauges),
2877                )
2878                .await;
2879                // Store the measurement before the task returns: quiescence
2880                // may be reached just as the deadline stops the join loop from
2881                // collecting results. Unfinished or failed waits report false.
2882                forwarding
2883                    .record_daemon_drain_result(
2884                        endpoint,
2885                        drained_after_quiescence_wait(&wait_result),
2886                    )
2887                    .map_err(SuperviseError::Forwarding)?;
2888                wait_result
2889            });
2890        }
2891        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2892            if !matches!(result, Ok(Ok(true))) {
2893                warn!(?result, "daemon shutdown drain did not reach quiescence");
2894            }
2895        }
2896        Ok(())
2897    }
2898
2899    /// The last step of an announced daemon shutdown, after the notice and the
2900    /// drain: send every registered module a module GOODBYE, the same planned
2901    /// stop signal `ck module stop` gives, then close every connection so each
2902    /// subc module sees EOF and starts its own teardown, then end every
2903    /// supervised child that has not exited
2904    /// by its own deadline (its drain budget, capped). Modules lead their own
2905    /// process groups, so a
2906    /// service manager's group kill no longer reaches them; without this a
2907    /// child that does not stop on EOF (every `protocol: "none"` child, which
2908    /// has no connection) would outlive the daemon. Every wait is bounded (see
2909    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2910    #[cfg(unix)]
2911    pub(crate) async fn end_children_for_daemon_shutdown(
2912        &self,
2913        already_escalated: bool,
2914        escalate: impl std::future::Future<Output = ()>,
2915    ) {
2916        tokio::pin!(escalate);
2917        let mut escalated = already_escalated;
2918        if let Some(forwarding) = &self.forwarding {
2919            let reason = CloseReason::new(
2920                "daemon_shutdown",
2921                "the daemon is exiting after its shutdown notice and drain",
2922            );
2923            if escalated {
2924                // The operator asked to stop waiting: queue the GOODBYEs but
2925                // do not wait for them to be written.
2926                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2927            } else {
2928                tokio::select! {
2929                    biased;
2930                    _ = escalate.as_mut() => {
2931                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2932                        escalated = true;
2933                    }
2934                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2935                }
2936            }
2937            let closed = forwarding.close_all_connections(&reason);
2938            debug!(closed, "closed established connections for daemon shutdown");
2939        }
2940        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2941        // already completed and must not be polled again; the child shutdown
2942        // wait is told it is escalated and gets a future that never fires.
2943        let escalated_here = escalated && !already_escalated;
2944        let remaining_escalate = async move {
2945            if escalated_here {
2946                std::future::pending::<()>().await;
2947            } else {
2948                escalate.await;
2949            }
2950        };
2951        crate::child_roster::end_children_for_daemon_shutdown(
2952            &self.child_roster,
2953            escalated,
2954            remaining_escalate,
2955        )
2956        .await;
2957    }
2958
2959    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2960        Self {
2961            registry,
2962            restart_policy,
2963            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2964            connection_file_path: None,
2965            capture_logs_dir: None,
2966            forwarding: None,
2967            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2968            supervisor_handle: None,
2969            health: HealthConfig::default(),
2970            daemon_start_clock: crate::clock::StartClock::capture(),
2971            terminal_journal: None,
2972            spawn_events: SpawnEventFeed::default(),
2973            provenance_probe: ExecutableIdentityProbe::default(),
2974            child_roster: ChildRoster::default(),
2975            #[cfg(target_os = "linux")]
2976            cgroup_placement: None,
2977            #[cfg(test)]
2978            test_after_first_spawn: AfterFirstSpawnHook::default(),
2979        }
2980    }
2981
2982    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2983        self.drain_timeout = drain_timeout;
2984        self
2985    }
2986
2987    pub fn with_process_liveness(
2988        mut self,
2989        process_liveness: Arc<SupervisorProcessLiveness>,
2990    ) -> Self {
2991        self.process_liveness = process_liveness;
2992        self
2993    }
2994
2995    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2996        self.connection_file_path = Some(connection_file_path.into());
2997        self
2998    }
2999
3000    /// Enables daemon-owned capture files for supervised stdout and stderr.
3001    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3002        self.capture_logs_dir = Some(logs_dir.into());
3003        self
3004    }
3005
3006    /// Names this daemon lifetime in spawn events, independently of whether a
3007    /// terminal journal is configured.
3008    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3009        // A millisecond start stamp can repeat after clock rollback or a rapid
3010        // restart. Use the connection file's random daemon_id instead: it already
3011        // identifies this daemon lifetime independently of the wall clock.
3012        self.spawn_events.configure_incarnation(daemon_incarnation);
3013        self
3014    }
3015
3016    /// Enables best-effort history shared by every supervised module. Without
3017    /// it, terminal history is kept only in each module's in-memory ring.
3018    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3019        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3020        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3021            path,
3022            daemon_incarnation,
3023        )));
3024        this
3025    }
3026
3027    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3028        self.forwarding = Some(forwarding);
3029        self
3030    }
3031
3032    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3033        self.spawn_events = supervisor_handle.spawn_events.clone();
3034        self.supervisor_handle = Some(supervisor_handle);
3035        self
3036    }
3037
3038    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3039        self.health = health;
3040        self
3041    }
3042
3043    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3044    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3045    /// record is kept.
3046    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3047        self.child_roster.record_to(path.into());
3048        self
3049    }
3050
3051    #[cfg(target_os = "linux")]
3052    pub fn with_cgroup_placement(
3053        mut self,
3054        cgroup_placement: Option<subc_cgroup::Placement>,
3055    ) -> Self {
3056        self.cgroup_placement = cgroup_placement;
3057        self
3058    }
3059
3060    /// Spawn `spec.program` and start monitoring it.
3061    ///
3062    /// The child is expected to parse `--subc <connection-file-path>`, read the
3063    /// TCP+key connection file, authenticate to the already-running listener, and
3064    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3065    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3066        validate_spec(&spec)?;
3067        self.establish_identity(&spec);
3068
3069        let runtime = self.runtime_config();
3070        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3071        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3072        let spawned = spawn_child(
3073            &spec,
3074            runtime.connection_file_path.as_deref(),
3075            self.supervisor_handle.as_ref(),
3076            &runtime.stderr_ring,
3077            runtime.capture_logs_dir.as_deref(),
3078            &runtime.child_roster,
3079            #[cfg(target_os = "linux")]
3080            runtime.cgroup_placement.as_ref(),
3081        );
3082        #[cfg(test)]
3083        self.test_after_first_spawn.run(&spec.module_id);
3084        let child = match spawned {
3085            Ok(child) => child,
3086            Err(err) => {
3087                // Unlike the configured paths, a failed `spawn` leaves nothing
3088                // on the roster, so the module must not stay marked configured.
3089                self.abandon_unrostered(&spec.module_id);
3090                return Err(err);
3091            }
3092        };
3093        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3094        if let Some(spawn) = activity_spawn {
3095            spawn.complete(&snapshot);
3096        }
3097
3098        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3099    }
3100
3101    /// Make `spec`'s module count as configured, with its identity gates
3102    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3103    /// exists.
3104    ///
3105    /// Every path that takes on a new module calls this before `spawn_child`.
3106    /// The order is the point: the child can connect, register, sync its
3107    /// scopes and ask about them as soon as it is spawned, and the module is
3108    /// only put on the roster after `spawn_child` returns. Were the mark set
3109    /// with the roster entry, a fast child would see its own owner reported
3110    /// as not configured, and a scoped `route.open` in that window would be
3111    /// refused as terminal `scope_not_live` ("will never sync") instead of
3112    /// retryable `scope_not_synced`.
3113    fn establish_identity(&self, spec: &ModuleSpec) {
3114        if let Some(supervisor_handle) = &self.supervisor_handle {
3115            supervisor_handle.apply_identity_configuration(spec);
3116            supervisor_handle.mark_configured(&spec.module_id);
3117        }
3118    }
3119
3120    /// Take back [`Self::establish_identity`]'s configured mark when the
3121    /// module will not be put on the roster after all.
3122    fn abandon_unrostered(&self, module_id: &str) {
3123        if let Some(supervisor_handle) = &self.supervisor_handle {
3124            supervisor_handle.unmark_configured_unless_rostered(module_id);
3125        }
3126    }
3127
3128    /// Record a freshly spawned first process as running. On failure the
3129    /// module never reaches the roster, so its configured mark is taken back.
3130    fn mark_first_process_running(
3131        &self,
3132        spec: &ModuleSpec,
3133        runtime: &SupervisorRuntimeConfig,
3134        snapshot: &SharedSnapshot,
3135        child: &SupervisedChild,
3136    ) -> Result<(), SuperviseError> {
3137        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3138            self.abandon_unrostered(&spec.module_id);
3139            return Err(err);
3140        }
3141        self.process_liveness
3142            .track(spec.module_id.clone(), Arc::clone(snapshot));
3143        Ok(())
3144    }
3145
3146    /// Start supervising a module declared in daemon configuration.
3147    ///
3148    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3149    /// failures in the supervisor handle so operator-facing `supervisor.list`
3150    /// reflects every configured module while daemon startup continues.
3151    pub fn supervise_configured(
3152        &self,
3153        spec: ModuleSpec,
3154        enabled: bool,
3155    ) -> Result<SupervisedModule, SuperviseError> {
3156        self.supervise_configured_requested(
3157            spec,
3158            enabled,
3159            self.health.clone(),
3160            None,
3161            self.restart_policy,
3162            None,
3163        )
3164    }
3165
3166    /// Supervise a configured module with its own health, drain, and crash
3167    /// budget. The restart policy is per-module because the config file is:
3168    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3169    /// module that is expensive to restart should not be forced onto the same
3170    /// budget as one that is cheap.
3171    pub fn supervise_configured_with_health(
3172        &self,
3173        spec: ModuleSpec,
3174        enabled: bool,
3175        health: HealthConfig,
3176        drain_timeout_ms: Option<u64>,
3177        restart_policy: RestartPolicy,
3178    ) -> Result<SupervisedModule, SuperviseError> {
3179        self.supervise_configured_requested(
3180            spec,
3181            enabled,
3182            health,
3183            drain_timeout_ms,
3184            restart_policy,
3185            None,
3186        )
3187    }
3188
3189    pub(crate) fn supervise_configured_requested(
3190        &self,
3191        spec: ModuleSpec,
3192        enabled: bool,
3193        health: HealthConfig,
3194        drain_timeout_ms: Option<u64>,
3195        restart_policy: RestartPolicy,
3196        requester: Option<Principal>,
3197    ) -> Result<SupervisedModule, SuperviseError> {
3198        let registered = self.register_configured_with_health(
3199            spec,
3200            enabled,
3201            health,
3202            drain_timeout_ms,
3203            restart_policy,
3204        )?;
3205        self.start_registered_configured(registered, requester)
3206    }
3207
3208    /// Record a configured module as starting or disabled, without launching a
3209    /// process or running its monitor. Bootstrap registers the entire roster
3210    /// before publishing discovery, then activates it once children can read
3211    /// the connection file passed in their `--subc` argument.
3212    pub(crate) fn register_configured_with_health(
3213        &self,
3214        spec: ModuleSpec,
3215        enabled: bool,
3216        health: HealthConfig,
3217        drain_timeout_ms: Option<u64>,
3218        restart_policy: RestartPolicy,
3219    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3220        validate_spec(&spec)?;
3221        self.establish_identity(&spec);
3222
3223        let mut runtime = self.runtime_config();
3224        runtime.health = health;
3225        runtime.restart_policy = restart_policy;
3226        if let Some(ms) = drain_timeout_ms {
3227            runtime.drain_timeout = Duration::from_millis(ms);
3228            *runtime
3229                .effective_drain_timeout
3230                .lock()
3231                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3232        }
3233        let snapshot = if enabled {
3234            SupervisorSnapshot::starting()
3235        } else {
3236            SupervisorSnapshot::disabled()
3237        };
3238        let snapshot = {
3239            let mut snapshot = snapshot;
3240            if enabled {
3241                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3242                {
3243                    snapshot.waiting_for_previous_pid = Some(pid);
3244                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3245                }
3246            }
3247            snapshot
3248        };
3249        let (module, commands) =
3250            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3251        Ok(RegisteredConfiguredModule {
3252            spec,
3253            runtime,
3254            module,
3255            commands,
3256        })
3257    }
3258
3259    /// Activate a previously registered module. Queued control requests are
3260    /// served by the monitor after the initial configured spawn has settled.
3261    pub(crate) fn start_registered_configured(
3262        &self,
3263        registered: RegisteredConfiguredModule,
3264        requester: Option<Principal>,
3265    ) -> Result<SupervisedModule, SuperviseError> {
3266        let RegisteredConfiguredModule {
3267            spec,
3268            mut runtime,
3269            module,
3270            commands,
3271        } = registered;
3272        runtime.start_requester = requester;
3273        let snapshot = &module.inner.snapshot;
3274        let should_spawn = {
3275            let state = lock_snapshot(snapshot)?;
3276            state.enabled && state.waiting_for_previous_pid.is_none()
3277        };
3278        let child = if should_spawn {
3279            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3280            let spawned = spawn_child(
3281                &spec,
3282                runtime.connection_file_path.as_deref(),
3283                self.supervisor_handle.as_ref(),
3284                &runtime.stderr_ring,
3285                runtime.capture_logs_dir.as_deref(),
3286                &runtime.child_roster,
3287                #[cfg(target_os = "linux")]
3288                runtime.cgroup_placement.as_ref(),
3289            );
3290            #[cfg(test)]
3291            self.test_after_first_spawn.run(&spec.module_id);
3292            match spawned {
3293                Ok(child) => {
3294                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3295                    if let Some(spawn) = activity_spawn {
3296                        spawn.complete(snapshot);
3297                    }
3298                    Some(child)
3299                }
3300                Err(err) => {
3301                    if runtime.health.critical {
3302                        error!(
3303                            module_id = %spec.module_id,
3304                            program = %spec.program.display(),
3305                            error = %err,
3306                            "critical configured module failed to spawn; marking failed and alerting"
3307                        );
3308                    } else {
3309                        error!(
3310                            module_id = %spec.module_id,
3311                            program = %spec.program.display(),
3312                            error = %err,
3313                            "configured module failed to spawn; marking failed and continuing"
3314                        );
3315                    }
3316                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3317                        state.state = ModuleState::Failed;
3318                        clear_current_process_facts(state);
3319                    })?;
3320                    None
3321                }
3322            }
3323        } else {
3324            None
3325        };
3326        self.start_monitor(spec, runtime, &module, child, commands);
3327        Ok(module)
3328    }
3329
3330    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3331        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3332        SupervisorRuntimeConfig {
3333            start_requester: None,
3334            pending_activity_restart: false,
3335            provenance_probe: self.provenance_probe.clone(),
3336            scheduled_respawn: Arc::default(),
3337            deferred_reload_reply: Arc::default(),
3338            restart_policy: self.restart_policy,
3339            drain_timeout: self.drain_timeout,
3340            // Shared with this module's roster copy: daemon shutdown waits on
3341            // each child for the module's own drain budget, as resolved now.
3342            child_roster: self
3343                .child_roster
3344                .for_module(Arc::clone(&effective_drain_timeout)),
3345            effective_drain_timeout,
3346            default_drain_timeout: self.drain_timeout,
3347            health: self.health.clone(),
3348            connection_file_path: self.connection_file_path.clone(),
3349            capture_logs_dir: self.capture_logs_dir.clone(),
3350            forwarding: self.forwarding.clone(),
3351            supervisor_handle: self.supervisor_handle.clone(),
3352            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3353            terminal_ring: Arc::new(Mutex::new(
3354                TerminalRing::new(
3355                    TerminalRingConfig::default(),
3356                    self.daemon_start_clock.started_at_ms(),
3357                )
3358                .with_start_clock(self.daemon_start_clock)
3359                .with_journal(self.terminal_journal.clone())
3360                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3361            )),
3362            spawn_events: self.spawn_events.clone(),
3363            #[cfg(target_os = "linux")]
3364            cgroup_placement: self.cgroup_placement.clone(),
3365            #[cfg(test)]
3366            test_seed_stale_facts_before_enable_spawn: false,
3367            #[cfg(test)]
3368            test_reload_exit_record_gate: None,
3369        }
3370    }
3371
3372    fn supervised_module(
3373        &self,
3374        spec: ModuleSpec,
3375        runtime: SupervisorRuntimeConfig,
3376        snapshot: SharedSnapshot,
3377        child: Option<SupervisedChild>,
3378    ) -> SupervisedModule {
3379        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3380        self.start_monitor(spec, runtime, &module, child, commands);
3381        module
3382    }
3383
3384    fn register_module(
3385        &self,
3386        spec: &ModuleSpec,
3387        runtime: &SupervisorRuntimeConfig,
3388        snapshot: SharedSnapshot,
3389    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3390        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3391            spec: spec.clone(),
3392            health: runtime.health.clone(),
3393        }));
3394        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3395        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3396        // The module's OWN policy, which may be its per-module config rather than
3397        // the supervisor-wide one; status must report the budget the supervise
3398        // loop actually enforces.
3399        let restart_policy = runtime.restart_policy;
3400        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3401        let (tx, rx) = mpsc::channel(4);
3402        let module_id = spec.module_id.clone();
3403        let module = SupervisedModule {
3404            inner: Arc::new(SupervisedModuleInner {
3405                module_id: module_id.clone(),
3406                registry: Arc::clone(&self.registry),
3407                snapshot,
3408                configuration,
3409                stderr_ring,
3410                terminal_ring,
3411                commands: tx,
3412                monitor: Mutex::new(None),
3413                restart_policy,
3414                effective_drain_timeout,
3415                provenance_probe: self.provenance_probe.clone(),
3416            }),
3417        };
3418        // Identity gates and the configured mark must precede the roster entry,
3419        // so a client never sees a module without its admission policy.
3420        if let Some(supervisor_handle) = &self.supervisor_handle {
3421            supervisor_handle.insert(module.clone());
3422        }
3423        (module, rx)
3424    }
3425
3426    fn start_monitor(
3427        &self,
3428        spec: ModuleSpec,
3429        runtime: SupervisorRuntimeConfig,
3430        module: &SupervisedModule,
3431        child: Option<SupervisedChild>,
3432        commands: mpsc::Receiver<SupervisorCommand>,
3433    ) {
3434        let monitor = tokio::spawn(supervise_loop(
3435            spec,
3436            runtime,
3437            Arc::clone(&self.registry),
3438            Arc::clone(&self.process_liveness),
3439            Arc::clone(&module.inner.snapshot),
3440            child,
3441            commands,
3442        ));
3443        *module
3444            .inner
3445            .monitor
3446            .lock()
3447            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3448    }
3449}
3450
3451impl Default for Supervisor {
3452    fn default() -> Self {
3453        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3454    }
3455}
3456
3457/// Handle to one supervised child process.
3458#[derive(Clone)]
3459pub struct SupervisedModule {
3460    inner: Arc<SupervisedModuleInner>,
3461}
3462
3463struct SupervisedModuleInner {
3464    module_id: String,
3465    registry: Arc<Registry>,
3466    snapshot: SharedSnapshot,
3467    configuration: Arc<Mutex<SupervisedConfiguration>>,
3468    stderr_ring: Arc<Mutex<StderrRing>>,
3469    terminal_ring: Arc<Mutex<TerminalRing>>,
3470    commands: mpsc::Sender<SupervisorCommand>,
3471    monitor: Mutex<Option<JoinHandle<()>>>,
3472    /// Copied from the supervisor's runtime config at spawn so `status()` can
3473    /// report the restart budget without reaching back into the supervisor. The
3474    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3475    restart_policy: RestartPolicy,
3476    effective_drain_timeout: Arc<Mutex<Duration>>,
3477    provenance_probe: ExecutableIdentityProbe,
3478}
3479
3480impl fmt::Debug for SupervisedModule {
3481    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3482        f.debug_struct("SupervisedModule")
3483            .field("module_id", &self.inner.module_id)
3484            .field("status", &self.status())
3485            .finish_non_exhaustive()
3486    }
3487}
3488
3489impl SupervisedModule {
3490    pub fn module_id(&self) -> &str {
3491        &self.inner.module_id
3492    }
3493
3494    /// Test-only: put one probe miss on the streak, the way
3495    /// `handle_health_probe_failure` does, so tests can assert what a later
3496    /// event does to the streak without driving the whole probe loop.
3497    #[cfg(test)]
3498    pub(crate) fn record_health_probe_failure_for_test(
3499        &self,
3500        detail: &str,
3501    ) -> Result<(), SuperviseError> {
3502        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3503            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3504            state.health.detail = Some(detail.to_string());
3505        })
3506    }
3507
3508    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3509        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3510    }
3511
3512    pub(crate) fn registration_end_reason(
3513        &self,
3514    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3515        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3516        Ok(match snapshot.state {
3517            ModuleState::Draining if snapshot.draining_to_replace => {
3518                Some(RegistrationEndReason::SupervisorRestart)
3519            }
3520            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3521            ModuleState::Restarting if snapshot.draining_to_replace => {
3522                Some(RegistrationEndReason::SupervisorRestart)
3523            }
3524            _ => None,
3525        })
3526    }
3527
3528    /// The module's retained stderr, newest lines last.
3529    ///
3530    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3531    /// module, `supervisor.list` renders every module, and putting it in the
3532    /// shared snapshot would make each status read carry a payload almost nobody
3533    /// asked for. Callers that want the text ask for it.
3534    pub fn stderr_tail(
3535        &self,
3536        max_lines: Option<usize>,
3537        max_bytes: Option<usize>,
3538    ) -> StderrTailSnapshot {
3539        self.inner
3540            .stderr_ring
3541            .lock()
3542            .unwrap_or_else(|poisoned| poisoned.into_inner())
3543            .snapshot(max_lines, max_bytes)
3544    }
3545
3546    /// The module's bounded terminal history, oldest retained exit first.
3547    ///
3548    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3549    /// daemon whose in-memory history was necessarily reset.
3550    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3551        self.inner
3552            .terminal_ring
3553            .lock()
3554            .unwrap_or_else(|poisoned| poisoned.into_inner())
3555            .snapshot()
3556    }
3557
3558    /// Retained observations from the current ring and all journal generations.
3559    ///
3560    /// Blocking: this reads the journal files. Async callers use
3561    /// [`Self::read_durable_terminal_history`].
3562    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3563        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3564    }
3565
3566    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3567    /// read (up to every retained generation) never occupies a runtime worker.
3568    /// Fails only if the blocking task could not finish (runtime shutdown or a
3569    /// panic in the read).
3570    pub(crate) async fn read_durable_terminal_history(
3571        &self,
3572    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3573        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3574        let module_id = self.inner.module_id.clone();
3575        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3576            .await
3577    }
3578
3579    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3580        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3581            .map(|(status, _)| status)
3582    }
3583
3584    pub(crate) fn record_deliberate_severance(
3585        &self,
3586        identity: ProcessIdentity,
3587    ) -> Result<bool, SuperviseError> {
3588        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3589        if snapshot.pid != Some(identity.pid)
3590            || snapshot.process_start_time != Some(identity.start_time)
3591        {
3592            return Ok(false);
3593        }
3594        snapshot.deliberate_severance = Some(identity);
3595        Ok(true)
3596    }
3597
3598    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3599    ///
3600    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3601    /// does not produce reader-observability logs.
3602    pub(crate) fn status_for_control(
3603        &self,
3604        caller: &'static str,
3605    ) -> Result<ModuleStatus, SuperviseError> {
3606        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3607            .map(|(status, _)| status)
3608    }
3609
3610    fn status_with_snapshot_lock(
3611        &self,
3612        snapshot: &SharedSnapshot,
3613        caller: Option<&'static str>,
3614    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3615        let mut guard = match caller {
3616            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3617            None => lock_snapshot(snapshot)?,
3618        };
3619        // Read the budget through the pruning path so a reader sees the same
3620        // in-window count the restart decision would use, not a stale total.
3621        let restart_count =
3622            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3623        let snapshot = guard.clone();
3624        drop(guard);
3625        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3626            SuperviseError::StatePoisoned {
3627                module_id: Some(self.inner.module_id.clone()),
3628            }
3629        })?;
3630        let registration_active = self
3631            .inner
3632            .registry
3633            .get_module(&self.inner.module_id)
3634            .map_err(SuperviseError::Registry)?
3635            .is_some();
3636        let protocol = snapshot
3637            .spawned_protocol
3638            .unwrap_or(self.declared_protocol()?);
3639        let running_process =
3640            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3641        // Registration is the difference between the two protocols and the only
3642        // one: a subc module that has not registered cannot serve a request even
3643        // though its process is up, and a `none` module never registers at all,
3644        // so requiring it there would pin `live` to false for the whole life of
3645        // a perfectly healthy process.
3646        let live = match protocol {
3647            ModuleProtocol::Subc => running_process && registration_active,
3648            ModuleProtocol::None => running_process,
3649        };
3650
3651        Ok((
3652            ModuleStatus {
3653                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3654                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3655                module_id: self.inner.module_id.clone(),
3656                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3657                state: snapshot.state,
3658                enabled: snapshot.enabled,
3659                process_alive: snapshot.process_alive,
3660                registration_active,
3661                protocol,
3662                live,
3663                restart_count,
3664                lifetime_restarts: snapshot.lifetime_restarts,
3665                spawn_generation: snapshot.spawn_generation,
3666                max_restarts: self.inner.restart_policy.max_restarts,
3667                restart_window: self.inner.restart_policy.window,
3668                drain_timeout,
3669                restart_backoff: self.inner.restart_policy.backoff,
3670                restart_max_backoff: self.inner.restart_policy.max_backoff,
3671                pid: snapshot.reported_pid(),
3672                spawned_at_ms: snapshot.spawned_at_ms,
3673                spawned_from: snapshot.spawned_from,
3674                launch_spec_digest: snapshot.launch_spec_digest,
3675                process_start_time: snapshot.process_start_time,
3676                last_exit: snapshot.last_exit,
3677                health: snapshot.health,
3678            },
3679            snapshot.spawned_file_identity.clone(),
3680        ))
3681    }
3682
3683    #[cfg(test)]
3684    pub(crate) fn hold_snapshot_for_test(
3685        &self,
3686        acquired: std::sync::mpsc::Sender<()>,
3687        hold: Duration,
3688    ) -> std::thread::JoinHandle<()> {
3689        let snapshot = Arc::clone(&self.inner.snapshot);
3690        std::thread::spawn(move || {
3691            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3692            acquired
3693                .send(())
3694                .expect("test receiver waits for snapshot lock");
3695            std::thread::sleep(hold);
3696        })
3697    }
3698
3699    /// The status and the running-image check for `supervisor.provenance`,
3700    /// taken from one status read. The exec acknowledgement can land between
3701    /// two separate reads, and the reply would then pair "no pid yet" with an
3702    /// image observed after the module started, which describes no single
3703    /// moment.
3704    pub(crate) async fn status_and_running_image_agreement(
3705        &self,
3706    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3707        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3708        let image = self
3709            .inner
3710            .provenance_probe
3711            .observe(
3712                status.pid,
3713                status.spawned_from.as_deref(),
3714                identity,
3715                status.process_start_time,
3716            )
3717            .await;
3718        Ok((status, image))
3719    }
3720
3721    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3722        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3723            Ok(snapshot) => snapshot.clone(),
3724            Err(_) => {
3725                return subc_control::RunningImageAgreement::Unavailable {
3726                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3727                };
3728            }
3729        };
3730        self.inner
3731            .provenance_probe
3732            .observe(
3733                snapshot.reported_pid(),
3734                snapshot.spawned_from.as_deref(),
3735                snapshot.spawned_file_identity.clone(),
3736                snapshot.process_start_time,
3737            )
3738            .await
3739    }
3740
3741    /// Memory and CPU time of the module's current process, read now. Only the
3742    /// process the supervisor spawned is read, not processes it has started.
3743    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3744        #[cfg(windows)]
3745        {
3746            let image = lock_snapshot(&self.inner.snapshot)
3747                .ok()
3748                .and_then(|snapshot| {
3749                    snapshot
3750                        .spawned_file_identity
3751                        .as_ref()
3752                        .and_then(|identity| identity.windows_image.clone())
3753                });
3754            match image {
3755                Some(image) => {
3756                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3757                }
3758                None => subc_control::ChildResourceUsage::Unavailable {
3759                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3760                },
3761            }
3762        }
3763        #[cfg(not(windows))]
3764        {
3765            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3766                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3767                Err(_) => {
3768                    return subc_control::ChildResourceUsage::Unavailable {
3769                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3770                    }
3771                }
3772            };
3773            crate::child_resources::read(pid, start_time)
3774        }
3775    }
3776
3777    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3778        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3779        Ok(match snapshot.state {
3780            ModuleState::Restarting => true,
3781            ModuleState::Failed | ModuleState::Disabled => false,
3782            _ if snapshot.enabled && snapshot.respawn_pending => true,
3783            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3784        })
3785    }
3786
3787    #[cfg(test)]
3788    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3789        self.is_warming_with_snapshot_lock(None)
3790    }
3791
3792    pub(crate) fn is_warming_for_control(
3793        &self,
3794        caller: &'static str,
3795    ) -> Result<bool, SuperviseError> {
3796        self.is_warming_with_snapshot_lock(Some(caller))
3797    }
3798
3799    fn is_warming_with_snapshot_lock(
3800        &self,
3801        caller: Option<&'static str>,
3802    ) -> Result<bool, SuperviseError> {
3803        let snapshot = match caller {
3804            Some(caller) => {
3805                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3806            }
3807            None => lock_snapshot(&self.inner.snapshot)?,
3808        }
3809        .clone();
3810        Ok(matches!(
3811            snapshot.state,
3812            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3813        ))
3814    }
3815
3816    /// Drain the module and stop monitoring it.
3817    pub async fn drain(&self) -> Result<(), SuperviseError> {
3818        self.stop().await
3819    }
3820
3821    pub(crate) async fn retire_requested(
3822        &self,
3823        requester: Option<Principal>,
3824    ) -> Result<(), SuperviseError> {
3825        match self.state()? {
3826            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3827            ModuleState::Starting
3828            | ModuleState::Running
3829            | ModuleState::Unresponsive
3830            | ModuleState::Restarting
3831            | ModuleState::Draining
3832            | ModuleState::Disabled => {}
3833        }
3834
3835        let (reply_tx, reply_rx) = oneshot::channel();
3836        self.inner
3837            .commands
3838            .send(SupervisorCommand::Retire {
3839                requester,
3840                reply: reply_tx,
3841            })
3842            .await
3843            .map_err(|_| SuperviseError::CommandClosed {
3844                module_id: self.inner.module_id.clone(),
3845            })?;
3846        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3847            module_id: self.inner.module_id.clone(),
3848        })?
3849    }
3850
3851    pub async fn stop(&self) -> Result<(), SuperviseError> {
3852        self.stop_requested(None).await
3853    }
3854
3855    pub(crate) async fn stop_requested(
3856        &self,
3857        requester: Option<Principal>,
3858    ) -> Result<(), SuperviseError> {
3859        match self.state()? {
3860            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3861            ModuleState::Starting
3862            | ModuleState::Running
3863            | ModuleState::Unresponsive
3864            | ModuleState::Restarting
3865            | ModuleState::Draining
3866            | ModuleState::Disabled => {}
3867        }
3868
3869        let (reply_tx, reply_rx) = oneshot::channel();
3870        self.inner
3871            .commands
3872            .send(SupervisorCommand::Drain {
3873                requester,
3874                reply: reply_tx,
3875            })
3876            .await
3877            .map_err(|_| SuperviseError::CommandClosed {
3878                module_id: self.inner.module_id.clone(),
3879            })?;
3880        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3881            module_id: self.inner.module_id.clone(),
3882        })?
3883    }
3884
3885    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3886        self.restart_requested(drain_timeout_ms, None).await
3887    }
3888
3889    pub(crate) async fn restart_requested(
3890        &self,
3891        drain_timeout_ms: Option<u64>,
3892        requester: Option<Principal>,
3893    ) -> Result<(), SuperviseError> {
3894        let queued_at = Instant::now();
3895        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3896        let (reply_tx, reply_rx) = oneshot::channel();
3897        self.inner
3898            .commands
3899            .send(SupervisorCommand::Restart {
3900                requester,
3901                drain_timeout_ms,
3902                received_at_generation,
3903                queued_at,
3904                reply: reply_tx,
3905            })
3906            .await
3907            .map_err(|_| SuperviseError::CommandClosed {
3908                module_id: self.inner.module_id.clone(),
3909            })?;
3910        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3911            module_id: self.inner.module_id.clone(),
3912        })?
3913    }
3914
3915    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3916    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3917    /// process then drains in the background of the supervise loop) or has
3918    /// failed, leaving the old process serving.
3919    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3920        self.swap_requested(ready_timeout, None).await
3921    }
3922
3923    pub(crate) async fn swap_requested(
3924        &self,
3925        ready_timeout: Option<Duration>,
3926        requester: Option<Principal>,
3927    ) -> Result<(), SuperviseError> {
3928        let (reply_tx, reply_rx) = oneshot::channel();
3929        self.inner
3930            .commands
3931            .send(SupervisorCommand::Swap {
3932                requester,
3933                ready_timeout,
3934                reply: reply_tx,
3935            })
3936            .await
3937            .map_err(|_| SuperviseError::CommandClosed {
3938                module_id: self.inner.module_id.clone(),
3939            })?;
3940        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3941            module_id: self.inner.module_id.clone(),
3942        })?
3943    }
3944
3945    pub async fn reload(&self) -> Result<(), SuperviseError> {
3946        self.reload_requested(None).await
3947    }
3948
3949    pub(crate) async fn reload_requested(
3950        &self,
3951        requester: Option<Principal>,
3952    ) -> Result<(), SuperviseError> {
3953        let (reply_tx, reply_rx) = oneshot::channel();
3954        self.inner
3955            .commands
3956            .send(SupervisorCommand::Reload {
3957                requester,
3958                reply: reply_tx,
3959            })
3960            .await
3961            .map_err(|_| SuperviseError::CommandClosed {
3962                module_id: self.inner.module_id.clone(),
3963            })?;
3964        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3965            module_id: self.inner.module_id.clone(),
3966        })?
3967    }
3968
3969    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3970        self.set_enabled_requested(enabled, None).await
3971    }
3972
3973    pub(crate) async fn set_enabled_requested(
3974        &self,
3975        enabled: bool,
3976        requester: Option<Principal>,
3977    ) -> Result<bool, SuperviseError> {
3978        let (reply_tx, reply_rx) = oneshot::channel();
3979        self.inner
3980            .commands
3981            .send(SupervisorCommand::SetEnabled {
3982                requester,
3983                enabled,
3984                reply: reply_tx,
3985            })
3986            .await
3987            .map_err(|_| SuperviseError::CommandClosed {
3988                module_id: self.inner.module_id.clone(),
3989            })?;
3990        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3991            module_id: self.inner.module_id.clone(),
3992        })?
3993    }
3994
3995    /// The current process's protocol, or the configured protocol when down.
3996    /// A rescan stores the next launch spec without changing how an existing
3997    /// process registers, serves routes, is probed, or exits.
3998    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3999        let configured = self
4000            .inner
4001            .configuration
4002            .lock()
4003            .map_err(|_| SuperviseError::StatePoisoned {
4004                module_id: Some(self.inner.module_id.clone()),
4005            })?
4006            .spec
4007            .protocol;
4008        let state = lock_snapshot(&self.inner.snapshot)?;
4009        Ok(state.spawned_protocol.unwrap_or(configured))
4010    }
4011
4012    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4013        let configuration =
4014            self.inner
4015                .configuration
4016                .lock()
4017                .map_err(|_| SuperviseError::StatePoisoned {
4018                    module_id: Some(self.inner.module_id.clone()),
4019                })?;
4020        Ok((configuration.spec.clone(), configuration.health.clone()))
4021    }
4022
4023    /// Replace this module's launch spec, keeping its health and drain policy,
4024    /// the way a rescan does for a changed config entry. The running process is
4025    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4026    #[cfg(any(test, feature = "test-support"))]
4027    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4028        let (_, health) = self.configuration()?;
4029        let drain_timeout_ms = u64::try_from(
4030            self.inner
4031                .effective_drain_timeout
4032                .lock()
4033                .unwrap_or_else(|poisoned| poisoned.into_inner())
4034                .as_millis(),
4035        )
4036        .ok();
4037        self.update_configuration(spec, health, drain_timeout_ms)
4038            .await
4039    }
4040
4041    pub(crate) async fn update_configuration(
4042        &self,
4043        spec: ModuleSpec,
4044        health: HealthConfig,
4045        drain_timeout_ms: Option<u64>,
4046    ) -> Result<(), SuperviseError> {
4047        if spec.module_id != self.inner.module_id {
4048            return Err(SuperviseError::InvalidSpec {
4049                reason: "a supervised module's module_id cannot be changed".to_string(),
4050            });
4051        }
4052        validate_spec(&spec)?;
4053        let (reply_tx, reply_rx) = oneshot::channel();
4054        self.inner
4055            .commands
4056            .send(SupervisorCommand::UpdateConfiguration {
4057                spec: spec.clone(),
4058                health: health.clone(),
4059                drain_timeout_ms,
4060                reply: reply_tx,
4061            })
4062            .await
4063            .map_err(|_| SuperviseError::CommandClosed {
4064                module_id: self.inner.module_id.clone(),
4065            })?;
4066        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4067            module_id: self.inner.module_id.clone(),
4068        })?;
4069        let mut configuration =
4070            self.inner
4071                .configuration
4072                .lock()
4073                .map_err(|_| SuperviseError::StatePoisoned {
4074                    module_id: Some(self.inner.module_id.clone()),
4075                })?;
4076        configuration.spec = spec;
4077        configuration.health = health;
4078        Ok(())
4079    }
4080}
4081
4082impl Drop for SupervisedModuleInner {
4083    fn drop(&mut self) {
4084        let Ok(mut monitor) = self.monitor.lock() else {
4085            return;
4086        };
4087        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4088            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4089                state.state = ModuleState::Stopped;
4090                clear_current_process_facts(state);
4091            });
4092            monitor.abort();
4093        }
4094        let _ = monitor.take();
4095    }
4096}
4097
4098#[derive(Debug)]
4099enum SupervisorCommand {
4100    Drain {
4101        requester: Option<Principal>,
4102        reply: oneshot::Sender<Result<(), SuperviseError>>,
4103    },
4104    Retire {
4105        requester: Option<Principal>,
4106        reply: oneshot::Sender<Result<(), SuperviseError>>,
4107    },
4108    Restart {
4109        requester: Option<Principal>,
4110        /// Operator override for this one restart's drain budget, in ms. `None`
4111        /// uses the module's configured/default budget; `Some(0)` cuts
4112        /// immediately (wedge bounce: a stuck request never settles, so
4113        /// waiting only delays recovery).
4114        drain_timeout_ms: Option<u64>,
4115        /// The module's `spawn_generation` when the request was received, before
4116        /// it waited in the command queue. A queued restart whose module has
4117        /// since spawned a newer process is already satisfied (see the handler).
4118        received_at_generation: u64,
4119        /// When the request entered the command queue, so the handler can log
4120        /// how long it waited behind the loop's other work.
4121        queued_at: Instant,
4122        reply: oneshot::Sender<Result<(), SuperviseError>>,
4123    },
4124    Reload {
4125        requester: Option<Principal>,
4126        reply: oneshot::Sender<Result<(), SuperviseError>>,
4127    },
4128    SetEnabled {
4129        requester: Option<Principal>,
4130        enabled: bool,
4131        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4132    },
4133    UpdateConfiguration {
4134        spec: ModuleSpec,
4135        health: HealthConfig,
4136        /// Per-module drain override from the new config; `None` re-resolves to
4137        /// the supervisor-wide default.
4138        drain_timeout_ms: Option<u64>,
4139        reply: oneshot::Sender<()>,
4140    },
4141    Swap {
4142        requester: Option<Principal>,
4143        /// How long the candidate may take to register and declare itself
4144        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4145        ready_timeout: Option<Duration>,
4146        /// Answered at cutover or failure; the incumbent's drain follows.
4147        reply: oneshot::Sender<Result<(), SuperviseError>>,
4148    },
4149}
4150
4151#[derive(Debug)]
4152pub enum SuperviseError {
4153    InvalidSpec {
4154        reason: String,
4155    },
4156    Spawn {
4157        program: PathBuf,
4158        source: io::Error,
4159        cgroup_path: Option<PathBuf>,
4160    },
4161    Cgroup {
4162        module_id: String,
4163        source: io::Error,
4164    },
4165    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4166    /// than spawn a reserved module without its identity binding.
4167    LaunchNonce {
4168        reason: String,
4169    },
4170    Wait {
4171        module_id: String,
4172        source: io::Error,
4173    },
4174    Kill {
4175        module_id: String,
4176        source: io::Error,
4177    },
4178    Forwarding(ForwardingError),
4179    Registry(RegistryError),
4180    ReloadUnavailable {
4181        module_id: String,
4182        reason: String,
4183    },
4184    /// An operator restart/reload was requested for a module that is currently
4185    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4186    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4187    /// by a restart, so these commands are rejected instead of re-enabling it.
4188    Disabled {
4189        module_id: String,
4190    },
4191    ReloadFailed {
4192        module_id: String,
4193        reason: String,
4194    },
4195    RegistrationStillActive {
4196        module_id: String,
4197        waited: Duration,
4198    },
4199    StatePoisoned {
4200        module_id: Option<String>,
4201    },
4202    CommandClosed {
4203        module_id: String,
4204    },
4205    /// A restart or reload arrived while a swap's candidate was warming. The
4206    /// swap owns the module until it cuts over or fails; a stop or disable
4207    /// would have aborted it instead.
4208    SwapInProgress {
4209        module_id: String,
4210    },
4211    /// A swap was refused before anything was spawned.
4212    SwapRefused {
4213        module_id: String,
4214        reason: SwapRefusal,
4215    },
4216    /// A swap spawned a candidate and gave up on it. The candidate has been
4217    /// killed and its slot freed; the incumbent was left serving and was never
4218    /// drained, except in the one `CutoverLost` case described on that arm.
4219    SwapFailed {
4220        module_id: String,
4221        arm: SwapFailureArm,
4222        detail: String,
4223        /// How the candidate exited, when it exited on its own before the
4224        /// supervisor gave up on it.
4225        candidate_exit: Option<ExitReport>,
4226    },
4227}
4228
4229/// Why a swap was refused before a candidate was spawned.
4230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4231pub enum SwapRefusal {
4232    /// The module's config does not declare `overlap: "safe"`.
4233    OverlapExclusive,
4234    /// The module is not registered, so there is no incumbent to keep serving
4235    /// and nothing a swap would improve on; a plain restart is the tool.
4236    NotRegistered,
4237    /// The module does not speak the subc wire, so a candidate could never
4238    /// register or declare itself ready.
4239    ProtocolNone,
4240    /// The supervisor lacks the forwarding table (to cut routes over) or the
4241    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4242    NotConfigured,
4243    /// A swap is already open for this module.
4244    AlreadySwapping,
4245}
4246
4247impl SwapRefusal {
4248    pub fn as_str(self) -> &'static str {
4249        match self {
4250            Self::OverlapExclusive => "overlap_exclusive",
4251            Self::NotRegistered => "not_registered",
4252            Self::ProtocolNone => "protocol_none",
4253            Self::NotConfigured => "not_configured",
4254            Self::AlreadySwapping => "already_swapping",
4255        }
4256    }
4257}
4258
4259/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4260/// serving and undrained; see `CutoverLost`.
4261#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4262pub enum SwapFailureArm {
4263    /// The candidate process could not be started.
4264    SpawnFailed,
4265    /// The candidate did not register within the readiness budget.
4266    NeverRegistered,
4267    /// The candidate registered but did not declare itself ready in time.
4268    NeverReady,
4269    /// The candidate exited before cutover.
4270    CandidateExited,
4271    /// The candidate declared itself ready but failed its health probe.
4272    CandidateUnhealthy,
4273    /// An operator stop, disable or retire arrived while the candidate warmed.
4274    /// The candidate was killed and the operator's command then carried out on
4275    /// the incumbent.
4276    Interrupted,
4277    /// The candidate's connection closed at the moment of cutover. If it
4278    /// closed before forwarding moved, the incumbent is untouched. If it closed
4279    /// between the forwarding and registry halves of cutover, forwarding can no
4280    /// longer route to the incumbent, so the module is restarted plainly.
4281    CutoverLost,
4282}
4283
4284impl SwapFailureArm {
4285    pub fn as_str(self) -> &'static str {
4286        match self {
4287            Self::SpawnFailed => "spawn_failed",
4288            Self::NeverRegistered => "never_registered",
4289            Self::NeverReady => "never_ready",
4290            Self::CandidateExited => "candidate_exited",
4291            Self::CandidateUnhealthy => "candidate_unhealthy",
4292            Self::Interrupted => "interrupted",
4293            Self::CutoverLost => "cutover_lost",
4294        }
4295    }
4296}
4297
4298impl fmt::Display for SuperviseError {
4299    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4300        match self {
4301            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4302            Self::Spawn {
4303                program,
4304                source,
4305                cgroup_path: Some(cgroup_path),
4306            } => write!(
4307                f,
4308                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4309                cgroup_path.display(),
4310                program.display()
4311            ),
4312            Self::Spawn {
4313                program,
4314                source,
4315                cgroup_path: None,
4316            } => write!(
4317                f,
4318                "failed to spawn module '{}': {source}",
4319                program.display()
4320            ),
4321            Self::Cgroup { module_id, source } => {
4322                write!(
4323                    f,
4324                    "failed to prepare cgroup for module '{module_id}': {source}"
4325                )
4326            }
4327            Self::LaunchNonce { reason } => {
4328                write!(
4329                    f,
4330                    "failed to generate reserved-module launch nonce: {reason}"
4331                )
4332            }
4333            Self::Wait { module_id, source } => {
4334                write!(f, "failed to wait for module '{module_id}': {source}")
4335            }
4336            Self::Kill { module_id, source } => {
4337                write!(f, "failed to kill module '{module_id}': {source}")
4338            }
4339            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4340            Self::Registry(err) => write!(f, "registry error: {err}"),
4341            Self::ReloadUnavailable { module_id, reason } => {
4342                write!(f, "reload unavailable for module '{module_id}': {reason}")
4343            }
4344            Self::Disabled { module_id } => {
4345                write!(
4346                    f,
4347                    "module '{module_id}' is disabled; enable it before restart or reload"
4348                )
4349            }
4350            Self::ReloadFailed { module_id, reason } => {
4351                write!(f, "reload failed for module '{module_id}': {reason}")
4352            }
4353            Self::RegistrationStillActive { module_id, waited } => write!(
4354                f,
4355                "module '{module_id}' registration remained active after waiting {waited:?}"
4356            ),
4357            Self::StatePoisoned { module_id } => match module_id {
4358                Some(module_id) => {
4359                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4360                }
4361                None => write!(f, "supervisor state was poisoned"),
4362            },
4363            Self::CommandClosed { module_id } => {
4364                write!(
4365                    f,
4366                    "supervisor command channel for module '{module_id}' is closed"
4367                )
4368            }
4369            Self::SwapInProgress { module_id } => write!(
4370                f,
4371                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4372            ),
4373            Self::SwapRefused { module_id, reason } => match reason {
4374                SwapRefusal::OverlapExclusive => write!(
4375                    f,
4376                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4377                ),
4378                SwapRefusal::NotRegistered => write!(
4379                    f,
4380                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4381                ),
4382                SwapRefusal::ProtocolNone => write!(
4383                    f,
4384                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4385                ),
4386                SwapRefusal::NotConfigured => write!(
4387                    f,
4388                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4389                ),
4390                SwapRefusal::AlreadySwapping => {
4391                    write!(f, "module '{module_id}' is already being swapped")
4392                }
4393            },
4394            Self::SwapFailed {
4395                module_id,
4396                arm,
4397                detail,
4398                ..
4399            } => write!(
4400                f,
4401                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4402                arm.as_str()
4403            ),
4404        }
4405    }
4406}
4407
4408impl Error for SuperviseError {
4409    fn source(&self) -> Option<&(dyn Error + 'static)> {
4410        match self {
4411            Self::Spawn { source, .. }
4412            | Self::Cgroup { source, .. }
4413            | Self::Wait { source, .. }
4414            | Self::Kill { source, .. } => Some(source),
4415            Self::Forwarding(err) => Some(err),
4416            Self::Registry(err) => Some(err),
4417            Self::LaunchNonce { .. }
4418            | Self::InvalidSpec { .. }
4419            | Self::ReloadUnavailable { .. }
4420            | Self::Disabled { .. }
4421            | Self::ReloadFailed { .. }
4422            | Self::RegistrationStillActive { .. }
4423            | Self::StatePoisoned { .. }
4424            | Self::CommandClosed { .. }
4425            | Self::SwapInProgress { .. }
4426            | Self::SwapRefused { .. }
4427            | Self::SwapFailed { .. } => None,
4428        }
4429    }
4430}
4431
4432pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4433    if spec.module_id.trim().is_empty() {
4434        return Err(SuperviseError::InvalidSpec {
4435            reason: "module_id must not be empty".to_string(),
4436        });
4437    }
4438
4439    Ok(())
4440}
4441
4442#[derive(Debug, Default)]
4443struct HealthProbeRuntime {
4444    configured_health: Option<HealthConfig>,
4445    registered_connection: Option<crate::ConnectionId>,
4446    advertised: bool,
4447    next_probe_at: Option<Instant>,
4448    probe_index: u64,
4449}
4450
4451fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4452    lock_snapshot(snapshot)
4453        .ok()
4454        .and_then(|state| state.spawned_protocol)
4455        .unwrap_or(spec.protocol)
4456}
4457
4458impl HealthProbeRuntime {
4459    fn refresh_registration(
4460        &mut self,
4461        spec: &ModuleSpec,
4462        runtime: &SupervisorRuntimeConfig,
4463        registry: &Registry,
4464        snapshot: &SharedSnapshot,
4465    ) {
4466        if self.configured_health.as_ref() != Some(&runtime.health) {
4467            self.configured_health = Some(runtime.health.clone());
4468            self.next_probe_at = None;
4469            self.registered_connection = None;
4470            self.probe_index = 0;
4471        }
4472        // A non-wire process never registers. Only an explicitly configured
4473        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4474        // health failure for that kind of process.
4475        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4476            self.registered_connection = None;
4477            self.advertised = runtime.health.http.is_some();
4478            if !self.advertised {
4479                self.next_probe_at = None;
4480                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4481                    let unknown = ModuleHealthStatus::default();
4482                    if state.health != unknown {
4483                        state.health = unknown;
4484                    }
4485                });
4486            } else if self.next_probe_at.is_none() {
4487                self.next_probe_at = Some(
4488                    Instant::now()
4489                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4490                );
4491            }
4492            return;
4493        }
4494
4495        let registration = match registry.get_module(&spec.module_id) {
4496            Ok(registration) => registration,
4497            Err(err) => {
4498                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4499                self.advertised = false;
4500                self.next_probe_at = None;
4501                return;
4502            }
4503        };
4504
4505        let Some(registration) = registration else {
4506            self.registered_connection = None;
4507            self.advertised = false;
4508            self.next_probe_at = None;
4509            return;
4510        };
4511
4512        let advertised = registration
4513            .control_ops
4514            .iter()
4515            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4516        if !advertised {
4517            self.registered_connection = Some(registration.connection_id);
4518            self.advertised = false;
4519            self.next_probe_at = None;
4520            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4521                if state.health.status != SupervisorHealthStatus::Unknown
4522                    || state.health.consecutive_failures != 0
4523                    || state.health.last_probe_ms.is_some()
4524                    || state.health.detail.is_some()
4525                    || state.health.metrics.is_some()
4526                {
4527                    state.health.status = SupervisorHealthStatus::Unknown;
4528                    state.health.consecutive_failures = 0;
4529                    state.health.last_probe_ms = None;
4530                    state.health.detail = None;
4531                    state.health.metrics = None;
4532                }
4533            });
4534            return;
4535        }
4536
4537        let reregistered = self.registered_connection != Some(registration.connection_id);
4538        self.registered_connection = Some(registration.connection_id);
4539        self.advertised = true;
4540        if reregistered || self.next_probe_at.is_none() {
4541            self.probe_index = 0;
4542            self.next_probe_at = Some(
4543                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4544            );
4545            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4546                state.health.status = SupervisorHealthStatus::Unknown;
4547                state.health.consecutive_failures = 0;
4548                state.health.detail = None;
4549                state.health.metrics = None;
4550            });
4551        }
4552    }
4553
4554    fn wake_after(&self) -> Option<Duration> {
4555        if !self.advertised {
4556            return None;
4557        }
4558        self.next_probe_at
4559            .map(|next| next.saturating_duration_since(Instant::now()))
4560    }
4561
4562    fn due(&self) -> bool {
4563        self.advertised
4564            && self
4565                .next_probe_at
4566                .is_some_and(|next| Instant::now() >= next)
4567    }
4568
4569    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4570        self.probe_index = self.probe_index.wrapping_add(1);
4571        self.next_probe_at = Some(
4572            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4573        );
4574    }
4575}
4576
4577/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4578///
4579/// This was a struct with a single `message: String`, and every one of the
4580/// fifteen construction sites collapsed into it. Each site knows exactly what it
4581/// saw -- the lane is gone, the module did not answer in time, the module
4582/// answered with the wrong thing -- and `handle_health_probe_failure` then
4583/// treated all of them identically: increment a counter, compare to a threshold,
4584/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4585/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4586///
4587/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4588///
4589/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4590///   answer on it again.
4591/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4592///   AND with a perfectly healthy one that lost a CPU race -- which is what
4593///   happens under machine load, and is how this supervisor killed a healthy
4594///   module three times in one day.
4595/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4596///   Restarting on it is defensible, but it is not the silence case and should
4597///   never be counted as one.
4598/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4599///   anything, so it cannot be evidence about the module at all.
4600///
4601/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4602/// one that fires most often, and while every variant collapsed into one string
4603/// it carried the same weight as the strongest.
4604///
4605/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4606/// DESIGN and a reader stopping at it gets the build backwards: the restart
4607/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4608/// probes still increment the failure streak and drive escalation at the
4609/// threshold (see `is_proof_of_death` below for why that is deliberate and
4610/// what gates the change). Absence of evidence restarts modules today.
4611#[derive(Debug)]
4612enum HealthProbeEvidence {
4613    /// The module's control lane is gone. Proof of death.
4614    LaneDead,
4615    /// No reply within the deadline. Proves nothing about the module's state.
4616    NoAnswer,
4617    /// The module replied, but not with a usable health report. Proves it is alive.
4618    BadAnswer,
4619    /// The daemon could not ask. Says nothing about the module.
4620    Misconfigured,
4621}
4622
4623#[derive(Debug)]
4624struct HealthProbeError {
4625    evidence: HealthProbeEvidence,
4626    message: String,
4627}
4628
4629impl HealthProbeError {
4630    fn lane_dead(message: impl Into<String>) -> Self {
4631        Self::with(HealthProbeEvidence::LaneDead, message)
4632    }
4633
4634    fn no_answer(message: impl Into<String>) -> Self {
4635        Self::with(HealthProbeEvidence::NoAnswer, message)
4636    }
4637
4638    fn bad_answer(message: impl Into<String>) -> Self {
4639        Self::with(HealthProbeEvidence::BadAnswer, message)
4640    }
4641
4642    fn misconfigured(message: impl Into<String>) -> Self {
4643        Self::with(HealthProbeEvidence::Misconfigured, message)
4644    }
4645
4646    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4647        Self {
4648            evidence,
4649            message: message.into(),
4650        }
4651    }
4652
4653    /// Whether this observation is proof the module cannot serve.
4654    ///
4655    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4656    /// variant that fires under CPU starvation, and treating it as proof is the
4657    /// defect this enum exists to make impossible to reintroduce silently.
4658    ///
4659    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4660    /// to restart also needs a bound for the case it excludes -- a genuinely
4661    /// wedged module, alive but never answering -- and that bound must come from
4662    /// the distribution of real late-answer latencies, which nothing measures
4663    /// yet. Landing the classification first makes the later change a one-line
4664    /// decision against evidence that already exists, rather than two unproven
4665    /// changes at once.
4666    #[allow(dead_code)]
4667    fn is_proof_of_death(&self) -> bool {
4668        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4669    }
4670
4671    /// Short stable label for logs and the health snapshot.
4672    ///
4673    /// An operator reading `ck health` currently cannot tell "the module is gone"
4674    /// from "the module did not answer in five seconds", because both render as
4675    /// prose in the same field. These labels are what make the two
4676    /// distinguishable at a glance, and they are what a later restart-policy
4677    /// change will be argued from.
4678    fn label(&self) -> &'static str {
4679        match self.evidence {
4680            HealthProbeEvidence::LaneDead => "lane-dead",
4681            HealthProbeEvidence::NoAnswer => "no-answer",
4682            HealthProbeEvidence::BadAnswer => "bad-answer",
4683            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4684        }
4685    }
4686}
4687
4688impl fmt::Display for HealthProbeError {
4689    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4690        f.write_str(&self.message)
4691    }
4692}
4693
4694async fn run_health_probe_cycle(
4695    spec: &ModuleSpec,
4696    runtime: &SupervisorRuntimeConfig,
4697    registry: &Registry,
4698    process_liveness: &SupervisorProcessLiveness,
4699    snapshot: &SharedSnapshot,
4700    child: &mut Option<SupervisedChild>,
4701) {
4702    let now_ms = unix_ms_now();
4703    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4704        .then_some(runtime.health.http.as_deref())
4705        .flatten();
4706    let result = match http {
4707        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4708        None => probe_module_health(&spec.module_id, runtime, None).await,
4709    };
4710    match result {
4711        Ok(report) => {
4712            handle_health_report(
4713                spec,
4714                runtime,
4715                registry,
4716                process_liveness,
4717                snapshot,
4718                child,
4719                report,
4720                now_ms,
4721            )
4722            .await;
4723        }
4724        Err(err) => {
4725            if http.is_some() {
4726                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4727                    state.health.status = SupervisorHealthStatus::Failing;
4728                });
4729            }
4730            handle_health_probe_failure(
4731                spec,
4732                runtime,
4733                registry,
4734                process_liveness,
4735                snapshot,
4736                child,
4737                err,
4738                now_ms,
4739            )
4740            .await;
4741        }
4742    }
4743}
4744
4745pub(crate) struct HttpProbeTarget<'a> {
4746    address: std::net::SocketAddr,
4747    localhost: bool,
4748    authority: &'a str,
4749    path: String,
4750}
4751
4752/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4753/// or TLS. A URL cannot turn a local health check into an outbound connection.
4754pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4755    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4756        return Err("must not contain whitespace, controls, or a fragment".into());
4757    }
4758    let rest = url
4759        .strip_prefix("http://")
4760        .ok_or("must use plain http://")?;
4761    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4762    let (authority, suffix) = rest.split_at(split);
4763    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4764        ("::1", rest)
4765    } else {
4766        let split = authority.find(':').unwrap_or(authority.len());
4767        authority.split_at(split)
4768    };
4769    let ip: std::net::IpAddr = match host {
4770        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4771        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4772        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4773    };
4774    let port = if port.is_empty() {
4775        80
4776    } else {
4777        port.strip_prefix(':')
4778            .and_then(|p| p.parse::<u16>().ok())
4779            .filter(|p| *p > 0)
4780            .ok_or("must have a valid nonzero TCP port")?
4781    };
4782    let path = if suffix.is_empty() {
4783        "/".into()
4784    } else if suffix.starts_with('?') {
4785        format!("/{suffix}")
4786    } else {
4787        suffix.into()
4788    };
4789    Ok(HttpProbeTarget {
4790        address: std::net::SocketAddr::new(ip, port),
4791        localhost: host == "localhost",
4792        authority,
4793        path,
4794    })
4795}
4796
4797async fn probe_http_health(
4798    url: &str,
4799    deadline: Duration,
4800) -> Result<HealthReport, HealthProbeError> {
4801    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4802    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4803    // Keep partial diagnostics outside the timed future so cancellation does
4804    // not discard a status line or body bytes already received.
4805    let mut response_status = String::new();
4806    let mut body = Vec::new();
4807    let probe = async {
4808        // Resolve localhost ourselves so a hosts-file override cannot turn
4809        // this into an outbound request, while IPv6-only local servers work.
4810        let connection = match tokio::net::TcpStream::connect(target.address).await {
4811            Err(_) if target.localhost => {
4812                tokio::net::TcpStream::connect((
4813                    std::net::Ipv6Addr::LOCALHOST,
4814                    target.address.port(),
4815                ))
4816                .await
4817            }
4818            result => result,
4819        };
4820        let mut stream = connection.map_err(|error| {
4821            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4822        })?;
4823        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4824            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4825        let mut reader = BufReader::new(stream);
4826        let mut budget = 16 * 1024;
4827        let status = http_line(&mut reader, &mut budget).await?;
4828        let mut words = status.split_ascii_whitespace();
4829        let version = words.next();
4830        let code = words
4831            .next()
4832            .filter(|word| word.len() == 3)
4833            .and_then(|word| word.parse::<u16>().ok());
4834        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4835            || !code.is_some_and(|code| (100..600).contains(&code))
4836        {
4837            return Err(HealthProbeError::bad_answer(format!(
4838                "invalid HTTP status: {status}"
4839            )));
4840        }
4841        let code = code.expect("validated status code");
4842        response_status = status.clone();
4843        let mut length = None;
4844        let mut chunked = false;
4845        loop {
4846            let line = http_line(&mut reader, &mut budget).await?;
4847            if line.is_empty() {
4848                break;
4849            }
4850            if let Some((name, value)) = line.split_once(':') {
4851                if name.eq_ignore_ascii_case("content-length") {
4852                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4853                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4854                    })?);
4855                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4856                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4857                }
4858            }
4859        }
4860        if chunked {
4861            while body.len() < 200 {
4862                let line = http_line(&mut reader, &mut budget).await?;
4863                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4864                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4865                if size == 0 {
4866                    break;
4867                }
4868                let count = size.min((200 - body.len()) as u64) as usize;
4869                let start = body.len();
4870                (&mut reader)
4871                    .take(count as u64)
4872                    .read_to_end(&mut body)
4873                    .await
4874                    .map_err(|error| {
4875                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4876                    })?;
4877                if body.len() - start != count {
4878                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4879                }
4880                if size > count as u64 || body.len() == 200 {
4881                    break;
4882                }
4883                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4884                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4885                }
4886            }
4887        } else {
4888            reader
4889                .take(length.unwrap_or(200).min(200))
4890                .read_to_end(&mut body)
4891                .await
4892                .map_err(|error| {
4893                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4894                })?;
4895        }
4896        if (200..300).contains(&code) {
4897            Ok(HealthReport::ok())
4898        } else {
4899            Err(HealthProbeError::bad_answer(
4900                "HTTP health endpoint returned non-2xx",
4901            ))
4902        }
4903    };
4904    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4905        Err(HealthProbeError::no_answer(format!(
4906            "HTTP probe timed out after {deadline:?}"
4907        )))
4908    });
4909    if let Err(error) = &mut result {
4910        if !response_status.is_empty() {
4911            error.message = format!(
4912                "{}; {response_status}: {}",
4913                error.message,
4914                String::from_utf8_lossy(&body)
4915            );
4916        }
4917    }
4918    result
4919}
4920
4921async fn http_line(
4922    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4923    remaining: &mut usize,
4924) -> Result<String, HealthProbeError> {
4925    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4926    let mut line = Vec::new();
4927    (&mut *reader)
4928        .take(*remaining as u64)
4929        .read_until(b'\n', &mut line)
4930        .await
4931        .map_err(|error| {
4932            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4933        })?;
4934    *remaining -= line.len();
4935    if !line.ends_with(b"\r\n") {
4936        return Err(HealthProbeError::bad_answer(
4937            "HTTP headers are incomplete or exceed 16 KiB",
4938        ));
4939    }
4940    line.truncate(line.len() - 2);
4941    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4942}
4943
4944async fn probe_module_health(
4945    module_id: &str,
4946    runtime: &SupervisorRuntimeConfig,
4947    drain_deadline: Option<Instant>,
4948) -> Result<HealthReport, HealthProbeError> {
4949    let Some(forwarding) = runtime.forwarding.as_ref() else {
4950        return Err(HealthProbeError::misconfigured(
4951            "supervisor was not configured with a forwarding table",
4952        ));
4953    };
4954    let probe_started_at = Instant::now();
4955    let mut deadline = probe_started_at + runtime.health.deadline;
4956    if let Some(drain_deadline) = drain_deadline {
4957        deadline = deadline.min(drain_deadline);
4958    }
4959    let pending = if drain_deadline.is_some() {
4960        forwarding.begin_drain_health_probe_rpc_for(
4961            module_id,
4962            MODULE_CONTROL_OP_HEALTH_CHECK,
4963            probe_started_at,
4964            deadline,
4965        )
4966    } else {
4967        forwarding.begin_health_probe_rpc_for(
4968            module_id,
4969            MODULE_CONTROL_OP_HEALTH_CHECK,
4970            probe_started_at,
4971            deadline,
4972        )
4973    }
4974    .map_err(|err| {
4975        // The endpoint is not registered, so there is no live control lane to
4976        // ask. That is the module being absent, not slow.
4977        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4978    })?;
4979    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4980}
4981
4982/// [`probe_module_health`] for one endpoint rather than the id's active one.
4983///
4984/// A swap probes two processes that no by-id lookup reaches: its candidate
4985/// before cutover, and its superseded incumbent (for busy gauges) while the
4986/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4987/// bounds the by-id drain probe.
4988async fn probe_endpoint_health(
4989    endpoint: crate::ModuleEndpointId,
4990    runtime: &SupervisorRuntimeConfig,
4991    deadline_cap: Option<Instant>,
4992) -> Result<HealthReport, HealthProbeError> {
4993    let Some(forwarding) = runtime.forwarding.as_ref() else {
4994        return Err(HealthProbeError::misconfigured(
4995            "supervisor was not configured with a forwarding table",
4996        ));
4997    };
4998    let probe_started_at = Instant::now();
4999    let mut deadline = probe_started_at + runtime.health.deadline;
5000    if let Some(cap) = deadline_cap {
5001        deadline = deadline.min(cap);
5002    }
5003    let pending = forwarding
5004        .begin_endpoint_health_probe_rpc_for(
5005            endpoint,
5006            MODULE_CONTROL_OP_HEALTH_CHECK,
5007            probe_started_at,
5008            deadline,
5009        )
5010        .map_err(|err| {
5011            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5012        })?;
5013    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5014}
5015
5016/// Send a begun health probe and classify its answer.
5017async fn await_health_probe(
5018    forwarding: &ForwardingTable,
5019    pending: PendingModuleControlRpc,
5020    deadline: Instant,
5021    probe_budget: Duration,
5022) -> Result<HealthReport, HealthProbeError> {
5023    let PendingModuleControlRpc {
5024        endpoint,
5025        module_sink,
5026        negotiated_ver,
5027        corr,
5028        receiver,
5029    } = pending;
5030    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5031        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5032    })?;
5033    let frame = Frame::build_with_version(
5034        negotiated_ver,
5035        FrameType::Request,
5036        control_flags(),
5037        0,
5038        0,
5039        corr,
5040        body,
5041    )
5042    .map_err(|err| {
5043        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5044    })?;
5045
5046    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5047    // blocks waiting for capacity when the module's egress queue is full, and an
5048    // unbounded await here freezes the whole supervision actor (it stops polling
5049    // Child::wait and supervisor commands), making the module unrecoverable
5050    // in-band. On timeout the probe fails like any transport failure.
5051    match timeout_at(deadline, module_sink.send(frame)).await {
5052        Ok(Ok(())) => {}
5053        Ok(Err(err)) => {
5054            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5055            // A closed sink means the module's egress channel is gone -- the
5056            // receiving half is dropped when its connection tears down. Proof.
5057            return Err(HealthProbeError::lane_dead(format!(
5058                "failed to send health.check: {err}"
5059            )));
5060        }
5061        Err(_elapsed) => {
5062            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5063            // A full egress queue means the module is not draining its socket, which
5064            // is consistent with a wedged module AND with one whose reader is merely
5065            // starved. Silence, not proof.
5066            return Err(HealthProbeError::no_answer(
5067                "health.check send timed out before enqueue (module egress full)",
5068            ));
5069        }
5070    }
5071
5072    match timeout_at(deadline, receiver).await {
5073        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5074        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5075        // and those prove it is alive even though the probe failed.
5076        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5077            response.health_report().ok_or_else(|| {
5078                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5079            })
5080        }
5081        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5082            format!("health.check rejected: {}", body.message),
5083        )),
5084        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5085            Err(HealthProbeError::lane_dead(message))
5086        }
5087        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5088            Err(HealthProbeError::bad_answer(message))
5089        }
5090        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5091            Err(HealthProbeError::bad_answer(format!(
5092                "expected module-control op '{expected}', got '{actual}'"
5093            )))
5094        }
5095        // A reply that crosses the deadline before this waiter observes it is
5096        // still proof of life. The forwarding path records its end-to-end latency
5097        // before delivering this classification.
5098        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5099            "module answered health.check after its daemon deadline",
5100        )),
5101        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5102            "health.check waiter was canceled before the module responded",
5103        )),
5104        Err(_) => {
5105            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5106            Err(HealthProbeError::no_answer(format!(
5107                "module did not answer health.check within {probe_budget:?}"
5108            )))
5109        }
5110    }
5111}
5112
5113#[allow(clippy::too_many_arguments)]
5114async fn handle_health_report(
5115    spec: &ModuleSpec,
5116    runtime: &SupervisorRuntimeConfig,
5117    registry: &Registry,
5118    process_liveness: &SupervisorProcessLiveness,
5119    snapshot: &SharedSnapshot,
5120    child: &mut Option<SupervisedChild>,
5121    report: HealthReport,
5122    now_ms: u64,
5123) {
5124    let status = supervisor_health_status(report.status);
5125    let detail = report.detail.clone();
5126    let metrics = truncate_health_metrics(report.metrics);
5127    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5128        state.health.status = status;
5129        state.health.last_probe_ms = Some(now_ms);
5130        state.health.detail = detail.clone();
5131        state.health.metrics = metrics.clone();
5132        state.health.consecutive_failures = 0;
5133    });
5134
5135    let action = match report.status {
5136        HealthStatus::Ok => return,
5137        HealthStatus::Degraded => runtime.health.on_degraded,
5138        HealthStatus::Failing => runtime.health.on_failing,
5139    };
5140    apply_l3_health_action(
5141        spec,
5142        runtime,
5143        registry,
5144        process_liveness,
5145        snapshot,
5146        child,
5147        status,
5148        detail.as_deref(),
5149        action,
5150        now_ms,
5151    )
5152    .await;
5153}
5154
5155#[allow(clippy::too_many_arguments)]
5156async fn handle_health_probe_failure(
5157    spec: &ModuleSpec,
5158    runtime: &SupervisorRuntimeConfig,
5159    registry: &Registry,
5160    process_liveness: &SupervisorProcessLiveness,
5161    snapshot: &SharedSnapshot,
5162    child: &mut Option<SupervisedChild>,
5163    err: HealthProbeError,
5164    now_ms: u64,
5165) {
5166    let threshold = runtime.health.failure_threshold.max(1);
5167    let mut failures = 0;
5168    // Carry the evidence class into the operator-visible detail. Without it,
5169    // "module did not answer within 5s" and "the control lane is gone" are two
5170    // prose strings in the same field, and the reader has to know the codebase to
5171    // tell which one is proof of anything.
5172    let detail = format!("[{}] {err}", err.label());
5173    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5174        // A failed wire probe invalidates the last report, even before the
5175        // restart threshold. HTTP probes already mark failures as Failing.
5176        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5177            state.health.status = SupervisorHealthStatus::Unknown;
5178        }
5179        state.health.last_probe_ms = Some(now_ms);
5180        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5181        state.health.detail = Some(detail.clone());
5182        state.health.metrics = None;
5183        failures = state.health.consecutive_failures;
5184    });
5185
5186    if failures < threshold {
5187        warn!(
5188            module_id = %spec.module_id,
5189            consecutive_failures = failures,
5190            threshold,
5191            evidence = err.label(),
5192            detail = %detail,
5193            "health.check probe failed"
5194        );
5195        return;
5196    }
5197
5198    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5199        state.state = ModuleState::Unresponsive;
5200        state.health.status = SupervisorHealthStatus::Unresponsive;
5201    });
5202    // The evidence class is logged at the kill site because this is the line an
5203    // operator reads after an unexplained restart. A streak of `no-answer` under
5204    // machine load is the known false-positive shape; a `lane-dead` is not.
5205    if runtime.health.critical {
5206        error!(
5207            module_id = %spec.module_id,
5208            status = "unresponsive",
5209            evidence = err.label(),
5210            detail = %detail,
5211            "critical module health alert"
5212        );
5213    } else {
5214        warn!(
5215            module_id = %spec.module_id,
5216            status = "unresponsive",
5217            evidence = err.label(),
5218            detail = %detail,
5219            "module health threshold breached"
5220        );
5221    }
5222    if let Err(err) = health_restart_child(
5223        spec,
5224        runtime,
5225        registry,
5226        process_liveness,
5227        snapshot,
5228        child,
5229        SupervisorHealthStatus::Unresponsive,
5230        Some(&detail),
5231        now_ms,
5232    )
5233    .await
5234    {
5235        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5236    }
5237}
5238
5239#[allow(clippy::too_many_arguments)]
5240async fn apply_l3_health_action(
5241    spec: &ModuleSpec,
5242    runtime: &SupervisorRuntimeConfig,
5243    registry: &Registry,
5244    process_liveness: &SupervisorProcessLiveness,
5245    snapshot: &SharedSnapshot,
5246    child: &mut Option<SupervisedChild>,
5247    status: SupervisorHealthStatus,
5248    detail: Option<&str>,
5249    action: HealthAction,
5250    now_ms: u64,
5251) {
5252    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5253    match action {
5254        HealthAction::Report => {
5255            info!(
5256                module_id = %spec.module_id,
5257                status = ?status,
5258                detail,
5259                "module reported non-ok health"
5260            );
5261        }
5262        HealthAction::Alert => {
5263            error!(
5264                module_id = %spec.module_id,
5265                status = ?status,
5266                detail,
5267                "module health alert"
5268            );
5269        }
5270        HealthAction::Restart => {
5271            if let Err(err) = health_restart_child(
5272                spec,
5273                runtime,
5274                registry,
5275                process_liveness,
5276                snapshot,
5277                child,
5278                status,
5279                detail,
5280                now_ms,
5281            )
5282            .await
5283            {
5284                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5285            }
5286        }
5287    }
5288}
5289
5290#[allow(clippy::too_many_arguments)]
5291async fn health_restart_child(
5292    spec: &ModuleSpec,
5293    runtime: &SupervisorRuntimeConfig,
5294    registry: &Registry,
5295    process_liveness: &SupervisorProcessLiveness,
5296    snapshot: &SharedSnapshot,
5297    child: &mut Option<SupervisedChild>,
5298    status: SupervisorHealthStatus,
5299    detail: Option<&str>,
5300    now_ms: u64,
5301) -> Result<(), SuperviseError> {
5302    let (enabled, schedule) = {
5303        let mut state = lock_snapshot(snapshot)?;
5304        let enabled = state.enabled;
5305        let schedule = if enabled {
5306            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5307        } else {
5308            None
5309        };
5310        (enabled, schedule)
5311    };
5312
5313    if !enabled {
5314        return Err(SuperviseError::Disabled {
5315            module_id: spec.module_id.clone(),
5316        });
5317    }
5318
5319    if schedule.is_none() {
5320        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5321        error!(
5322            module_id = %spec.module_id,
5323            status = ?status,
5324            detail,
5325            max_restarts = runtime.restart_policy.max_restarts,
5326            window_secs = runtime.restart_policy.window.as_secs(),
5327            reason = %runtime.restart_policy.budget_exhausted_detail(),
5328            "health restart budget exhausted; marking module failed"
5329        );
5330        let stop_notice = begin_forwarding_drain_if_configured(
5331            spec,
5332            runtime,
5333            registry,
5334            snapshot,
5335            Some(true),
5336            RouteCloseReason::Disable,
5337        )
5338        .await?;
5339        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5340            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5341        })?;
5342        drain_optional_child(
5343            &spec.module_id,
5344            spec.protocol,
5345            stop_notice,
5346            registry,
5347            runtime.forwarding.as_deref(),
5348            snapshot,
5349            &runtime.terminal_ring,
5350            &runtime.spawn_events,
5351            child,
5352            runtime.drain_timeout,
5353            ModuleState::Failed,
5354            Some(true),
5355        )
5356        .await?;
5357        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5358        return Ok(());
5359    }
5360
5361    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5362    let mut restart_count = 0;
5363    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5364        restart_count = state.crash_restarts.len();
5365        state.state = ModuleState::Unresponsive;
5366        state.health.status = status;
5367        state.health.last_action = Some(HealthAction::Restart.to_string());
5368        state.health.last_action_ms = Some(now_ms);
5369    })?;
5370    warn!(
5371        module_id = %spec.module_id,
5372        status = ?status,
5373        detail,
5374        restart_count,
5375        restart_in_window = schedule.restart_in_window,
5376        delay_ms = schedule.delay.as_millis() as u64,
5377        "health-triggered module restart"
5378    );
5379
5380    let stop_notice = begin_forwarding_drain_if_configured(
5381        spec,
5382        runtime,
5383        registry,
5384        snapshot,
5385        Some(true),
5386        RouteCloseReason::Restart,
5387    )
5388    .await?;
5389    drain_optional_child(
5390        &spec.module_id,
5391        spec.protocol,
5392        stop_notice,
5393        registry,
5394        runtime.forwarding.as_deref(),
5395        snapshot,
5396        &runtime.terminal_ring,
5397        &runtime.spawn_events,
5398        child,
5399        runtime.drain_timeout,
5400        ModuleState::Restarting,
5401        Some(true),
5402    )
5403    .await?;
5404    schedule_respawn(
5405        runtime,
5406        snapshot,
5407        &spec.module_id,
5408        schedule.delay,
5409        RespawnKind::Spawn,
5410    )
5411}
5412
5413fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5414    if let Some(reply) = runtime
5415        .deferred_reload_reply
5416        .lock()
5417        .unwrap_or_else(|p| p.into_inner())
5418        .take()
5419    {
5420        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5421            module_id: module_id.to_string(),
5422            reason: reason.to_string(),
5423        }));
5424    }
5425}
5426
5427fn schedule_respawn(
5428    runtime: &SupervisorRuntimeConfig,
5429    snapshot: &SharedSnapshot,
5430    module_id: &str,
5431    delay: Duration,
5432    kind: RespawnKind,
5433) -> Result<(), SuperviseError> {
5434    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5435    update_snapshot(snapshot, Some(module_id), |state| {
5436        state.respawn_pending = true
5437    })?;
5438    *runtime
5439        .scheduled_respawn
5440        .lock()
5441        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5442        deadline: Instant::now() + delay,
5443        kind,
5444    });
5445    Ok(())
5446}
5447
5448fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5449    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5450        state.health.last_action = Some(action);
5451        state.health.last_action_ms = Some(now_ms);
5452    });
5453}
5454
5455fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5456    match status {
5457        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5458        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5459        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5460    }
5461}
5462
5463/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5464/// returned to every `supervisor.list` and `supervisor.health` caller.
5465///
5466/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5467/// path: that request exists to return a module's complete metrics object, and
5468/// `ck health <module-id>` documents it as the way to see what the cached view
5469/// truncates. The asymmetry is the feature.
5470///
5471/// So a new caller must decide which side it is on rather than assume the cap is
5472/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5473/// the truncation that path exists to avoid.
5474fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5475    let metrics = metrics?;
5476    match serde_json::to_vec(&metrics) {
5477        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5478            "truncated": true,
5479            "original_bytes": encoded.len(),
5480        })),
5481        Ok(_) | Err(_) => Some(metrics),
5482    }
5483}
5484
5485/// Spread health probes so a fleet-wide restart does not converge them.
5486///
5487/// The delay is derived from the module id and probe index rather than a random
5488/// source, so it is deterministic per module: a module keeps its own offset
5489/// across daemon restarts instead of re-rolling into a collision.
5490fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5491    if cadence.is_zero() {
5492        return Duration::ZERO;
5493    }
5494    let cadence_ms = cadence.as_millis() as u64;
5495    // This early return is REDUNDANT, deliberately, and a mutation run will show
5496    // it surviving removal. Recording why here so the next person to notice does
5497    // not have to re-derive it:
5498    //
5499    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5500    //   a zero cadence and builds the Duration from whole milliseconds, so a
5501    //   sub-millisecond cadence cannot come from config.
5502    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5503    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5504    //   -- exactly what this returns.
5505    //
5506    // Kept as a guard against a future widening of the config parser (accepting
5507    // microseconds, say), which would make the sub-millisecond case reachable.
5508    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5509    // divides by zero. Remove this and nothing changes.
5510    if cadence_ms == 0 {
5511        return cadence;
5512    }
5513    // Note that this never returns less than one cadence, including for the FIRST
5514    // probe. So a freshly registered module reports health `unknown` for a full
5515    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5516    // ready to answer.
5517    //
5518    // That is a property of the supervisor's schedule, not of any module: an
5519    // operator watching a restart sees `unknown` and cannot tell it from a module
5520    // that is slow to warm. Measured on two unrelated modules, both flipping to
5521    // `ok` between 22s and 32s after restart.
5522    //
5523    // Left as-is because spreading the first probe is what keeps a fleet-wide
5524    // restart from firing fourteen simultaneous probes into a cold machine. The
5525    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5526    // that thundering herd for a faster first reading.
5527    let jitter_span = (cadence_ms / 10).max(1);
5528    let hash = module_id.as_bytes().iter().fold(
5529        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5530        |acc, byte| {
5531            acc.wrapping_mul(1099511628211)
5532                .wrapping_add(u64::from(*byte))
5533        },
5534    );
5535    cadence + Duration::from_millis(hash % jitter_span)
5536}
5537
5538#[cfg(test)]
5539mod tests {
5540    use super::*;
5541
5542    #[test]
5543    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5544        let handle = SupervisorHandle::new();
5545        let module_id = "readded-tombstone";
5546        handle.record_rescan_removal(module_id);
5547        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5548
5549        handle.apply_identity_configuration(&ModuleSpec {
5550            module_id: module_id.to_string(),
5551            program: PathBuf::from("/test/module"),
5552            args: Vec::new(),
5553            env: Vec::new(),
5554            reserved: false,
5555            reserved_prefixes: Vec::new(),
5556            protocol: ModuleProtocol::Subc,
5557            overlap: Default::default(),
5558        });
5559
5560        assert!(
5561            handle.removal_tombstone_age_ms(module_id).is_none(),
5562            "a re-added module must not retain a stale removal tombstone"
5563        );
5564    }
5565
5566    /// What one module's owner looked like from the control plane at the
5567    /// instant after its first process was spawned.
5568    #[derive(Debug, PartialEq, Eq)]
5569    struct OwnerInSpawnWindow {
5570        module_id: String,
5571        configured: bool,
5572        on_roster: bool,
5573        admission_refusal: Option<&'static str>,
5574    }
5575
5576    /// A supervised module's process can connect, register, sync its scopes
5577    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5578    /// the module on the roster until afterward; configured supervision does
5579    /// so before spawning. In both cases the owner must
5580    /// already read as configured, so a scoped `route.open` against it is
5581    /// refused as retryable `scope_not_synced` and not as terminal
5582    /// `scope_not_live` ("will never sync").
5583    ///
5584    /// The hook runs just after each initial spawn attempt, before returning to
5585    /// the caller, so no race with a real child's registration is needed.
5586    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5587    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5588        use crate::scopes::ScopeTable;
5589        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5590
5591        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5592        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5593            module_id: module_id.to_string(),
5594            program,
5595            args: Vec::new(),
5596            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5597                .into_iter()
5598                .map(|key| (key.to_string(), dir.path().display().to_string()))
5599                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5600                .collect(),
5601            reserved: false,
5602            reserved_prefixes: Vec::new(),
5603            protocol: ModuleProtocol::Subc,
5604            overlap: Default::default(),
5605        };
5606        let live = super::terminal_history_tests::fake_aft_stub_path();
5607        let missing = dir.path().join("definitely-missing-module");
5608
5609        let handle = SupervisorHandle::new();
5610        let mut supervisor =
5611            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5612                .with_handle(handle.clone());
5613        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5614        let hook_handle = handle.clone();
5615        let hook_observed = Arc::clone(&observed);
5616        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5617            // Exactly what the control plane computes for a scoped route.open
5618            // naming this module as the owner of a scope it has not synced.
5619            let configured = hook_handle.is_configured(module_id);
5620            let selector = ScopeSelector::new(
5621                Principal::Reserved {
5622                    module_id: module_id.to_string(),
5623                },
5624                "s".to_string(),
5625                Some(1),
5626            );
5627            let carrier = Principal::Reserved {
5628                module_id: "carrier".to_string(),
5629            };
5630            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5631                .admit(&carrier, module_id, &selector, configured)
5632            {
5633                Ok(_) => None,
5634                Err(refusal) => Some(refusal.code),
5635            };
5636            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5637                module_id: module_id.to_string(),
5638                configured,
5639                on_roster: hook_handle.get(module_id).is_some(),
5640                admission_refusal,
5641            });
5642        })));
5643
5644        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5645        let configured = supervisor
5646            .supervise_configured(stub("configured", live.clone()), true)
5647            .unwrap();
5648        let with_health = supervisor
5649            .supervise_configured_with_health(
5650                stub("with-health", live.clone()),
5651                true,
5652                HealthConfig::default(),
5653                None,
5654                RestartPolicy::default(),
5655            )
5656            .unwrap();
5657        // The failed-spawn path still puts the module on the roster (as
5658        // failed), so it is configured throughout.
5659        let failed = supervisor
5660            .supervise_configured_with_health(
5661                stub("failed-spawn", missing.clone()),
5662                true,
5663                HealthConfig::default(),
5664                None,
5665                RestartPolicy::default(),
5666            )
5667            .unwrap();
5668        // A failed plain `spawn` puts nothing on the roster, so its mark is
5669        // taken back once the spawn has failed.
5670        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5671
5672        let expected = [
5673            "plain",
5674            "configured",
5675            "with-health",
5676            "failed-spawn",
5677            "spawn-error",
5678        ]
5679        .into_iter()
5680        .map(|module_id| OwnerInSpawnWindow {
5681            module_id: module_id.to_string(),
5682            configured: true,
5683            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5684            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5685        })
5686        .collect::<Vec<_>>();
5687        assert_eq!(*observed.lock().unwrap(), expected);
5688
5689        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5690            assert!(
5691                handle.get(module_id).is_some(),
5692                "{module_id} is on the roster"
5693            );
5694            assert!(
5695                handle.is_configured(module_id),
5696                "{module_id} stays configured"
5697            );
5698        }
5699        assert!(handle.get("spawn-error").is_none());
5700        assert!(
5701            !handle.is_configured("spawn-error"),
5702            "a plain spawn that failed must not leave its module marked configured"
5703        );
5704
5705        // Leaving the roster clears the mark with it.
5706        handle.retire("failed-spawn");
5707        assert!(!handle.is_configured("failed-spawn"));
5708
5709        for module in [plain, configured, with_health] {
5710            module.stop().await.unwrap();
5711        }
5712        drop(failed);
5713    }
5714
5715    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5716        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5717        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5718            snapshot.process_alive = true;
5719            snapshot.pid = Some(41);
5720            snapshot.spawned_at_ms = Some(42);
5721            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5722            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5723            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5724                device: 43,
5725                inode: 44,
5726                #[cfg(windows)]
5727                windows_image: None,
5728            });
5729        })
5730        .unwrap();
5731        snapshot
5732    }
5733
5734    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5735        let snapshot = lock_snapshot(snapshot).unwrap();
5736        assert!(!snapshot.process_alive);
5737        assert_eq!(snapshot.pid, None);
5738        assert_eq!(snapshot.spawned_at_ms, None);
5739        assert_eq!(snapshot.spawned_from, None);
5740        assert_eq!(snapshot.launch_spec_digest, None);
5741        assert_eq!(snapshot.spawned_file_identity, None);
5742    }
5743
5744    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5745    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5746        let supervisor =
5747            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5748        let mut runtime = supervisor.runtime_config();
5749        runtime.test_seed_stale_facts_before_enable_spawn = true;
5750        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5751        let mut child = None;
5752        let spec = ModuleSpec {
5753            module_id: "failed-enable-clears-facts".to_string(),
5754            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5755            args: Vec::new(),
5756            env: Vec::new(),
5757            reserved: false,
5758            reserved_prefixes: Vec::new(),
5759            protocol: ModuleProtocol::Subc,
5760            overlap: Default::default(),
5761        };
5762
5763        let result = set_child_enabled(
5764            &spec,
5765            &runtime,
5766            &supervisor.registry,
5767            &supervisor.process_liveness,
5768            &snapshot,
5769            &mut child,
5770            true,
5771        )
5772        .await;
5773
5774        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5775        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5776        assert_snapshot_process_facts_cleared(&snapshot);
5777    }
5778
5779    #[tokio::test]
5780    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5781        let supervisor =
5782            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5783        let runtime = supervisor.runtime_config();
5784        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5785            ModuleState::Restarting,
5786            true,
5787        )));
5788        let spec = ModuleSpec {
5789            module_id: "start-stranded-restarting".to_string(),
5790            program: super::terminal_history_tests::fake_aft_stub_path(),
5791            args: Vec::new(),
5792            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5793            reserved: false,
5794            reserved_prefixes: Vec::new(),
5795            protocol: ModuleProtocol::None,
5796            overlap: Default::default(),
5797        };
5798        let mut child = None;
5799        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5800        assert!(!super::set_child_enabled(
5801            &spec,
5802            &runtime,
5803            &Registry::default(),
5804            &supervisor.process_liveness,
5805            &snapshot,
5806            &mut child,
5807            true
5808        )
5809        .await
5810        .unwrap());
5811        assert!(child.is_none());
5812        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5813        assert!(super::set_child_enabled(
5814            &spec,
5815            &runtime,
5816            &Registry::default(),
5817            &supervisor.process_liveness,
5818            &snapshot,
5819            &mut child,
5820            true
5821        )
5822        .await
5823        .unwrap());
5824        assert_eq!(
5825            lock_snapshot(&snapshot).unwrap().state,
5826            ModuleState::Running
5827        );
5828        let mut child = child.unwrap();
5829        child.start_kill().unwrap();
5830        child.wait().await.unwrap();
5831    }
5832
5833    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5834    async fn failed_reload_spawn_clears_current_process_facts() {
5835        let supervisor =
5836            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5837        let mut runtime = supervisor.runtime_config();
5838        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5839        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5840        let mut child = None;
5841        let spec = ModuleSpec {
5842            module_id: "failed-reload-clears-facts".to_string(),
5843            program: PathBuf::from("/unused/failed-reload-module"),
5844            args: Vec::new(),
5845            env: Vec::new(),
5846            reserved: false,
5847            reserved_prefixes: Vec::new(),
5848            protocol: ModuleProtocol::Subc,
5849            overlap: Default::default(),
5850        };
5851
5852        let result = handle_reload_spawn_failure(
5853            &spec,
5854            &runtime,
5855            &supervisor.process_liveness,
5856            &snapshot,
5857            &mut child,
5858            "forced reload spawn failure".to_string(),
5859        )
5860        .await;
5861
5862        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5863        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5864        assert_snapshot_process_facts_cleared(&snapshot);
5865    }
5866
5867    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5868    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5869        let supervisor =
5870            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5871        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5872        let module = supervisor.supervised_module(
5873            ModuleSpec {
5874                module_id: "drop-clears-facts".to_string(),
5875                program: PathBuf::from("/unused/drop-module"),
5876                args: Vec::new(),
5877                env: Vec::new(),
5878                reserved: false,
5879                reserved_prefixes: Vec::new(),
5880                protocol: ModuleProtocol::Subc,
5881                overlap: Default::default(),
5882            },
5883            supervisor.runtime_config(),
5884            Arc::clone(&snapshot),
5885            None,
5886        );
5887        assert!(!module
5888            .inner
5889            .monitor
5890            .lock()
5891            .unwrap()
5892            .as_ref()
5893            .unwrap()
5894            .is_finished());
5895
5896        drop(module);
5897
5898        assert_eq!(
5899            lock_snapshot(&snapshot).unwrap().state,
5900            ModuleState::Stopped
5901        );
5902        assert_snapshot_process_facts_cleared(&snapshot);
5903    }
5904
5905    #[cfg(unix)]
5906    #[tokio::test]
5907    async fn rescan_preserves_running_protocol_until_respawn() {
5908        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5909        let initial = ModuleSpec {
5910            module_id: "rescan-protocol".into(),
5911            program: PathBuf::from("/bin/sleep"),
5912            args: vec!["60".into()],
5913            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5914                .into_iter()
5915                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5916                .collect(),
5917            reserved: false,
5918            reserved_prefixes: vec![],
5919            protocol: ModuleProtocol::None,
5920            overlap: Default::default(),
5921        };
5922        let supervisor =
5923            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5924        let module = supervisor.spawn(initial.clone()).unwrap();
5925        assert!(module.status().unwrap().live);
5926        let mut next = initial;
5927        next.protocol = ModuleProtocol::Subc;
5928        module
5929            .update_configuration(next.clone(), HealthConfig::default(), None)
5930            .await
5931            .unwrap();
5932        assert!(
5933            module.status().unwrap().live,
5934            "rescan must not require HELLO from the old non-wire process"
5935        );
5936        let runtime = supervisor.runtime_config();
5937        let action = on_child_exit(
5938            &next,
5939            RestartPolicy::default(),
5940            &supervisor.registry,
5941            &module.inner.snapshot,
5942            &runtime.terminal_ring,
5943            &runtime.spawn_events,
5944            &runtime.child_roster,
5945            ExitReport {
5946                kind: ExitKind::Clean,
5947                code: Some(0),
5948                signal: None,
5949                at_ms: unix_ms_now(),
5950            },
5951        )
5952        .await;
5953        assert!(
5954            matches!(action, NextAction::Restart { .. }),
5955            "the old non-wire process's clean exit must restart"
5956        );
5957        module.drain().await.unwrap();
5958    }
5959
5960    #[cfg(unix)]
5961    #[tokio::test]
5962    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5963        use std::os::unix::fs::PermissionsExt;
5964        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5965        let script = dir.join("module.sh");
5966        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5967        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5968        let record_path = dir.join("live-children.json");
5969        let supervisor =
5970            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5971                .with_live_children_record(&record_path);
5972        for (program, args) in [
5973            (PathBuf::from("sleep"), vec!["60".into()]),
5974            (script, vec![]),
5975        ] {
5976            let spec = ModuleSpec {
5977                module_id: "image-identity".into(),
5978                program: program.clone(),
5979                args,
5980                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5981                    .into_iter()
5982                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5983                    .collect(),
5984                reserved: false,
5985                reserved_prefixes: vec![],
5986                protocol: ModuleProtocol::None,
5987                overlap: Default::default(),
5988            };
5989            let module = supervisor.spawn(spec).unwrap();
5990            #[cfg(target_os = "macos")]
5991            {
5992                // SETEXEC confirmation is asynchronous; the orphan record must
5993                // identify the final image, never the intermediate trampoline.
5994                let deadline = Instant::now() + Duration::from_secs(5);
5995                while crate::live_children::read_record(&record_path)
5996                    .unwrap()
5997                    .iter()
5998                    .all(|entry| entry.executable.is_none())
5999                {
6000                    assert!(Instant::now() < deadline, "module image was not confirmed");
6001                    tokio::time::sleep(Duration::from_millis(5)).await;
6002                }
6003            }
6004            let entry = crate::live_children::read_record(&record_path)
6005                .unwrap()
6006                .pop()
6007                .unwrap();
6008            let observed = subc_os::Process::open(entry.pid)
6009                .unwrap()
6010                .unwrap()
6011                .observe()
6012                .unwrap();
6013            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6014            module.drain().await.unwrap();
6015            assert_eq!(
6016                verdict,
6017                crate::live_children::IdentityVerdict::Matches,
6018                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6019            );
6020        }
6021    }
6022
6023    #[cfg(unix)]
6024    fn http_fixture(
6025        dir: &std::path::Path,
6026        url: &str,
6027        threshold: u32,
6028    ) -> crate::daemon_config::ConfiguredModule {
6029        let path = dir.join("subc.jsonc");
6030        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6031            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6032            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6033            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6034            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6035        }}}).to_string()).unwrap();
6036        crate::daemon_config::load(&path)
6037            .unwrap()
6038            .unwrap()
6039            .modules
6040            .pop()
6041            .unwrap()
6042    }
6043
6044    #[cfg(unix)]
6045    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6046        timeout(Duration::from_secs(5), async {
6047            loop {
6048                if module.status().unwrap().health.status == status {
6049                    break;
6050                }
6051                sleep(Duration::from_millis(5)).await;
6052            }
6053        })
6054        .await
6055        .unwrap_or_else(|_| {
6056            panic!(
6057                "expected {status:?}, got {:?}",
6058                module.status().unwrap().health
6059            )
6060        });
6061    }
6062
6063    #[cfg(unix)]
6064    #[tokio::test]
6065    async fn http_health_status_flips_ok_failing_ok() {
6066        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6067        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6068        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6069        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6070        let serving_status = status.clone();
6071        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6072        let server = tokio::spawn(async move {
6073            loop {
6074                let (mut stream, _) = listener.accept().await.unwrap();
6075                let mut request = [0u8; 2048];
6076                let count = stream.read(&mut request).await.unwrap();
6077                assert!(count > 0, "a probe must send an HTTP request");
6078                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6079                let body = if code == 200 {
6080                    "ready"
6081                } else {
6082                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6083                };
6084                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6085                let _ = stream.write_all(response.as_bytes()).await;
6086            }
6087        });
6088        let configured = http_fixture(&dir, &url, 1000);
6089        let module =
6090            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6091                .supervise_configured_with_health(
6092                    configured.module_spec(),
6093                    true,
6094                    configured.health,
6095                    configured.drain_timeout_ms,
6096                    configured.restart,
6097                )
6098                .unwrap();
6099        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6100        status.store(503, std::sync::atomic::Ordering::SeqCst);
6101        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6102        assert!(module
6103            .status()
6104            .unwrap()
6105            .health
6106            .detail
6107            .unwrap()
6108            .contains("scratch failure"));
6109        status.store(200, std::sync::atomic::Ordering::SeqCst);
6110        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6111        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6112        let before = module.status().unwrap();
6113        let (spec, mut health) = module.configuration().unwrap();
6114        health.http = None;
6115        module
6116            .update_configuration(spec.clone(), health.clone(), Some(10))
6117            .await
6118            .unwrap();
6119        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6120        health.http = Some(url);
6121        module
6122            .update_configuration(spec, health, Some(10))
6123            .await
6124            .unwrap();
6125        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6126        assert_eq!(
6127            module.status().unwrap().pid,
6128            before.pid,
6129            "changing a probe must apply live, not restart its process"
6130        );
6131        let (spec, mut health) = module.configuration().unwrap();
6132        health.failure_threshold = 2;
6133        module
6134            .update_configuration(spec, health, Some(10))
6135            .await
6136            .unwrap();
6137        status.store(503, std::sync::atomic::Ordering::SeqCst);
6138        timeout(Duration::from_secs(5), async {
6139            while module.status().unwrap().spawn_generation == before.spawn_generation {
6140                sleep(Duration::from_millis(5)).await;
6141            }
6142        })
6143        .await
6144        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6145        module.drain().await.unwrap();
6146        server.abort();
6147    }
6148
6149    #[cfg(unix)]
6150    #[tokio::test]
6151    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6152        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6153        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6154        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6155        drop(unused);
6156        let configured = http_fixture(&dir, &url, 2);
6157        let module =
6158            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6159                .supervise_configured_with_health(
6160                    configured.module_spec(),
6161                    true,
6162                    configured.health,
6163                    configured.drain_timeout_ms,
6164                    configured.restart,
6165                )
6166                .unwrap();
6167        let before = module.status().unwrap().spawn_generation;
6168        timeout(Duration::from_secs(5), async {
6169            loop {
6170                let status = module.status().unwrap();
6171                if status.spawn_generation > before {
6172                    assert!(status.lifetime_restarts > 0);
6173                    break;
6174                }
6175                sleep(Duration::from_millis(5)).await;
6176            }
6177        })
6178        .await
6179        .expect("sustained HTTP refusal must trigger the health restart policy");
6180        module.drain().await.unwrap();
6181    }
6182
6183    #[cfg(unix)]
6184    #[tokio::test]
6185    async fn http_health_timeout_honours_deadline() {
6186        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6187        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6188        let server = tokio::spawn(async move {
6189            let _held = listener.accept().await.unwrap();
6190            std::future::pending::<()>().await;
6191        });
6192        let error = timeout(
6193            Duration::from_secs(1),
6194            probe_http_health(&url, Duration::from_millis(10)),
6195        )
6196        .await
6197        .expect("the probe must enforce its own deadline")
6198        .unwrap_err();
6199        server.abort();
6200        assert!(error.to_string().contains("timed out"));
6201    }
6202
6203    #[cfg(unix)]
6204    #[tokio::test]
6205    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6206        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6207        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6208        let url = format!(
6209            "http://localhost:{}/healthz",
6210            listener.local_addr().unwrap().port()
6211        );
6212        let server = tokio::spawn(async move {
6213            let (mut stream, _) = listener.accept().await.unwrap();
6214            let mut request = [0u8; 2048];
6215            assert!(stream.read(&mut request).await.unwrap() > 0);
6216            stream
6217                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6218                .await
6219                .unwrap();
6220        });
6221        // The deadline only bounds a hang. A probe that never tried the IPv6
6222        // address would be refused on 127.0.0.1 and fail at once, so a longer
6223        // deadline does not weaken the assertion; one second timed out under a
6224        // loaded parallel test run.
6225        assert_eq!(
6226            probe_http_health(&url, Duration::from_secs(10))
6227                .await
6228                .unwrap()
6229                .status,
6230            HealthStatus::Ok
6231        );
6232        server.await.unwrap();
6233    }
6234
6235    #[cfg(unix)]
6236    #[tokio::test]
6237    async fn http_health_timeout_keeps_partial_status_and_body() {
6238        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6239        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6240        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6241        let server = tokio::spawn(async move {
6242            let (mut stream, _) = listener.accept().await.unwrap();
6243            let mut request = [0u8; 2048];
6244            assert!(stream.read(&mut request).await.unwrap() > 0);
6245            stream
6246                .write_all(
6247                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6248                )
6249                .await
6250                .unwrap();
6251            std::future::pending::<()>().await;
6252        });
6253        let error = probe_http_health(&url, Duration::from_secs(1))
6254            .await
6255            .unwrap_err()
6256            .to_string();
6257        server.abort();
6258        assert!(
6259            error.contains("timed out")
6260                && error.contains("503 Unavailable")
6261                && error.contains("partial diagnostic"),
6262            "{error}"
6263        );
6264    }
6265
6266    #[cfg(unix)]
6267    #[tokio::test]
6268    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6269        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6270        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6271        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6272        let server = tokio::spawn(async move {
6273            let (mut stream, _) = listener.accept().await.unwrap();
6274            let mut request = [0u8; 2048];
6275            assert!(stream.read(&mut request).await.unwrap() > 0);
6276            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6277            let response = format!(
6278                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6279                body.len()
6280            );
6281            stream.write_all(response.as_bytes()).await.unwrap();
6282        });
6283        let error = probe_http_health(&url, Duration::from_secs(1))
6284            .await
6285            .unwrap_err()
6286            .to_string();
6287        server.await.unwrap();
6288        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6289        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6290        assert!(!error.contains("not-in-diagnostic"));
6291    }
6292
6293    #[cfg(unix)]
6294    #[tokio::test]
6295    async fn http_health_real_nats_server_monitoring() {
6296        if std::process::Command::new("nats-server")
6297            .arg("--version")
6298            .env("XDG_DATA_HOME", std::env::temp_dir())
6299            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6300            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6301            .output()
6302            .is_err()
6303        {
6304            eprintln!(
6305                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6306            );
6307            return;
6308        }
6309        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6310        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6311        let port = monitor.local_addr().unwrap().port();
6312        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6313        let client_port = client.local_addr().unwrap().port();
6314        let config = dir.join("server.conf");
6315        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6316        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6317        configured.program = PathBuf::from("nats-server");
6318        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6319        drop(monitor);
6320        drop(client);
6321        let module =
6322            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6323                .supervise_configured_with_health(
6324                    configured.module_spec(),
6325                    true,
6326                    configured.health,
6327                    configured.drain_timeout_ms,
6328                    configured.restart,
6329                )
6330                .unwrap();
6331        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6332        module.drain().await.unwrap();
6333    }
6334
6335    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6336    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6337        let supervisor =
6338            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6339        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6340        let initial = ModuleSpec {
6341            module_id: "rescan-preserves-spawn-facts".to_string(),
6342            program: PathBuf::from("/spawned/module"),
6343            args: Vec::new(),
6344            env: Vec::new(),
6345            reserved: false,
6346            reserved_prefixes: Vec::new(),
6347            protocol: ModuleProtocol::Subc,
6348            overlap: Default::default(),
6349        };
6350        let module = supervisor.supervised_module(
6351            initial.clone(),
6352            supervisor.runtime_config(),
6353            snapshot,
6354            None,
6355        );
6356        let before = module.status().unwrap();
6357        let mut replacement = initial;
6358        replacement.program = PathBuf::from("/rescanned/replacement-module");
6359
6360        module
6361            .update_configuration(replacement, HealthConfig::default(), None)
6362            .await
6363            .unwrap();
6364
6365        let after = module.status().unwrap();
6366        assert_eq!(after.pid, before.pid);
6367        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6368        assert_eq!(after.spawned_from, before.spawned_from);
6369        drop(module);
6370    }
6371}
6372
6373fn unix_ms_now() -> u64 {
6374    SystemTime::now()
6375        .duration_since(UNIX_EPOCH)
6376        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6377        .unwrap_or(0)
6378}
6379
6380async fn supervise_loop(
6381    mut spec: ModuleSpec,
6382    mut runtime: SupervisorRuntimeConfig,
6383    registry: Arc<Registry>,
6384    process_liveness: Arc<SupervisorProcessLiveness>,
6385    snapshot: SharedSnapshot,
6386    mut child: Option<SupervisedChild>,
6387    mut commands: mpsc::Receiver<SupervisorCommand>,
6388) {
6389    let mut health_probe = HealthProbeRuntime::default();
6390    // Registry writes (including embedded callers) notify this module only.
6391    // Subscribe before the first refresh; watch retains changes that arrive
6392    // while commands or probes are running. No polling fallback is needed.
6393    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6394        Ok(changes) => Some(changes),
6395        Err(err) => {
6396            // A poisoned registry cannot accept further writes, so it cannot
6397            // recover via a timer. Keep exit and command handling alive.
6398            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6399            None
6400        }
6401    };
6402    // All restart backoffs run here, including health and operator requests.
6403    // While one is pending the loop serves commands, so disable or drain can
6404    // cancel the replacement without spawning a process just to stop it.
6405    let mut pending_respawn: Option<PendingRespawn> = None;
6406    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6407    // before anything else so a stop that interrupted a swap runs at once.
6408    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6409    let mut previous_instance_recheck = tokio::time::interval_at(
6410        Instant::now() + Duration::from_secs(5),
6411        Duration::from_secs(5),
6412    );
6413    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6414    loop {
6415        #[cfg(test)]
6416        {
6417            lock_snapshot(&snapshot).unwrap().actor_select = None;
6418        }
6419        #[cfg(target_os = "macos")]
6420        if let Some(active) = child.as_mut() {
6421            active.confirm_privacy_exec().await;
6422        }
6423        if let Some(scheduled) = runtime
6424            .scheduled_respawn
6425            .lock()
6426            .unwrap_or_else(|p| p.into_inner())
6427            .take()
6428        {
6429            pending_respawn = Some(scheduled);
6430        }
6431        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6432            pending_respawn = None;
6433            cancel_deferred_reload(
6434                &runtime,
6435                &spec.module_id,
6436                "respawn cancelled by a supervisor command",
6437            );
6438        }
6439        if child.is_none()
6440            && pending_respawn.is_none()
6441            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6442        {
6443            cancel_deferred_reload(
6444                &runtime,
6445                &spec.module_id,
6446                "respawn cancelled before a replacement was spawned",
6447            );
6448            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6449                state.respawn_pending = false;
6450                state.coalesced_restart_pending = false;
6451                if matches!(
6452                    state.state,
6453                    ModuleState::Restarting
6454                        | ModuleState::Starting
6455                        | ModuleState::Draining
6456                        | ModuleState::Unresponsive
6457                ) {
6458                    error!(module_id = %spec.module_id, state = ?state.state,
6459                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6460                    state.state = ModuleState::Failed;
6461                    clear_current_process_facts(state);
6462                }
6463            });
6464        }
6465        if let Some(command) = requeued.pop_front() {
6466            if !handle_supervisor_command(
6467                command,
6468                &mut spec,
6469                &mut runtime,
6470                &registry,
6471                &process_liveness,
6472                &snapshot,
6473                &mut child,
6474                &mut commands,
6475                &mut requeued,
6476            )
6477            .await
6478            {
6479                return;
6480            }
6481            if child.is_some() || !respawn_still_pending(&snapshot) {
6482                pending_respawn = None;
6483                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6484                    state.respawn_pending = false
6485                });
6486            }
6487            continue;
6488        }
6489        if child.is_some() {
6490            #[cfg(test)]
6491            {
6492                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6493            }
6494            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6495            let wake_after = health_probe.wake_after();
6496            let probe_sleep = async move {
6497                match wake_after {
6498                    Some(delay) => sleep(delay).await,
6499                    None => std::future::pending().await,
6500                }
6501            };
6502            tokio::pin!(probe_sleep);
6503            // A `protocol: "none"` child (a plain process that never registers
6504            // over the subc wire, such as nats-server) with no HTTP health
6505            // check has nothing to probe, so it waits only for its exit or a
6506            // supervisor command. A rescan that adds an HTTP check is a command.
6507            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6508            #[cfg(test)]
6509            {
6510                let registration_pending = wire_child
6511                    && registration_changes
6512                        .as_ref()
6513                        .is_some_and(|changes| changes.has_changed().unwrap());
6514                let mut state = lock_snapshot(&snapshot).unwrap();
6515                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6516                    generation: state.spawn_generation,
6517                    turn: state.actor_turns,
6518                    registered_connection: health_probe.registered_connection,
6519                    next_probe_at: health_probe.next_probe_at,
6520                    wake_after,
6521                });
6522            }
6523            let active_child = child.as_mut().expect("child checked above");
6524            tokio::select! {
6525                wait_result = active_child.wait() => {
6526                    #[cfg(test)]
6527                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6528                    // Every arm below that gives up on the CHILD must keep the
6529                    // supervision task itself alive (child = None, loop
6530                    // continues into command-serving mode). Returning here
6531                    // closes the command channel, which makes the module
6532                    // permanently unrestartable in-band: a clean child exit
6533                    // of an enabled module once wedged the fleet this way
6534                    // ('supervisor command channel is closed') and required a
6535                    // full daemon restart to recover.
6536                    let exit_report = match wait_result {
6537                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6538                        Err(err) => {
6539                            active_child.drain_stderr(&spec.module_id).await;
6540                            fail_after_wait_error(
6541                                &spec.module_id,
6542                                &snapshot,
6543                                &runtime.terminal_ring,
6544                                &runtime.spawn_events,
6545                                format!("failed to wait for supervised module: {err}"),
6546                            );
6547                            untrack_if_registration_released(
6548                                &process_liveness,
6549                                &registry,
6550                                &spec.module_id,
6551                                &snapshot,
6552                            );
6553                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6554                            child = None;
6555                            continue;
6556                        }
6557                    };
6558                    active_child.drain_stderr(&spec.module_id).await;
6559
6560                    let next = on_child_exit(
6561                        &spec,
6562                        runtime.restart_policy,
6563                        &registry,
6564                        &snapshot,
6565                        &runtime.terminal_ring,
6566                        &runtime.spawn_events,
6567                        &runtime.child_roster,
6568                        exit_report,
6569                    ).await;
6570                    runtime.start_requester = None;
6571                    runtime.pending_activity_restart = false;
6572                    // The exit is recorded, so a daemon shutdown may stop
6573                    // waiting for this child (see `SupervisedChild::wait`).
6574                    active_child.release_roster();
6575                    match next {
6576                        NextAction::Stop { registration_released } => {
6577                            if registration_released {
6578                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6579                            }
6580                            child = None;
6581                        }
6582                        NextAction::Restart { schedule } => {
6583                            let delay = schedule.map_or(
6584                                runtime.restart_policy.delay_for_restart(0),
6585                                |schedule| schedule.delay,
6586                            );
6587                            if let Some(schedule) = schedule {
6588                                log_crash_respawn(&spec.module_id, schedule);
6589                            }
6590                            // The exited child is fully recorded at this point,
6591                            // so release it and count the backoff down in the
6592                            // command-serving branch below rather than sleeping
6593                            // here: commands cannot be received from inside this
6594                            // select arm, and an operator disable or drain that
6595                            // arrives during the backoff must cancel the pending
6596                            // respawn instead of waiting for it to spawn first.
6597                            child = None;
6598                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6599                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6600                        }
6601                    }
6602                }
6603                command = commands.recv() => {
6604                    #[cfg(test)]
6605                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6606                    let Some(command) = command else {
6607                        return;
6608                    };
6609                    if !handle_supervisor_command(
6610                        command,
6611                        &mut spec,
6612                        &mut runtime,
6613                        &registry,
6614                        &process_liveness,
6615                        &snapshot,
6616                        &mut child,
6617                        &mut commands,
6618                        &mut requeued,
6619                    ).await {
6620                        return;
6621                    }
6622                }
6623                _ = async {
6624                    match registration_changes.as_mut() {
6625                        Some(changes) => { let _ = changes.changed().await; }
6626                        None => std::future::pending().await,
6627                    }
6628                }, if wire_child => {
6629                    #[cfg(test)]
6630                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6631                }
6632                _ = &mut probe_sleep => {
6633                    #[cfg(test)]
6634                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6635                    if health_probe.due() {
6636                        run_health_probe_cycle(
6637                            &spec,
6638                            &runtime,
6639                            &registry,
6640                            &process_liveness,
6641                            &snapshot,
6642                            &mut child,
6643                        ).await;
6644                        if child.is_none() {
6645                            runtime.start_requester = None;
6646                            runtime.pending_activity_restart = false;
6647                        }
6648                        if child.is_some() {
6649                            health_probe.schedule_next(&spec, runtime.health.cadence);
6650                        }
6651                    }
6652                }
6653            }
6654        } else if let Some(pending) = pending_respawn {
6655            tokio::select! {
6656                _ = sleep_until(pending.deadline) => {
6657                    pending_respawn = None;
6658                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6659                    // A command handled below while the backoff elapsed may
6660                    // have stopped the module; never respawn past an operator's
6661                    // disable or drain.
6662                    if !respawn_still_pending(&snapshot) {
6663                        continue;
6664                    }
6665                    // The daemon began shutting down during the backoff: the
6666                    // spawn would be refused anyway, and refusing it here
6667                    // leaves the module stopped instead of reporting a
6668                    // failed restart.
6669                    if runtime.child_roster.is_closed() {
6670                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6671                            state.state = ModuleState::Stopped;
6672                        });
6673                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6674                        continue;
6675                    }
6676                    if let Err(err) = release_dead_registration(
6677                        &registry,
6678                        runtime.forwarding.as_deref(),
6679                        &snapshot,
6680                        &spec.module_id,
6681                    ).await {
6682                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6683                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6684                        continue;
6685                    }
6686
6687                    if matches!(pending.kind, RespawnKind::Reload) {
6688                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6689                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6690                        if result.is_ok() && runtime.pending_activity_restart {
6691                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6692                        }
6693                        runtime.pending_activity_restart = false;
6694                        if let Some(reply) = reply { let _ = reply.send(result); }
6695                        continue;
6696                    }
6697                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6698                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6699                        Ok(next_child) => {
6700                            child = Some(next_child);
6701                            if runtime.pending_activity_restart {
6702                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6703                                runtime.pending_activity_restart = false;
6704                            }
6705                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6706                        }
6707                        Err(err) => {
6708                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6709                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6710                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6711                        }
6712                    }
6713                }
6714                command = commands.recv() => {
6715                    let Some(command) = command else {
6716                        return;
6717                    };
6718                    if !handle_supervisor_command(
6719                        command,
6720                        &mut spec,
6721                        &mut runtime,
6722                        &registry,
6723                        &process_liveness,
6724                        &snapshot,
6725                        &mut child,
6726                        &mut commands,
6727                        &mut requeued,
6728                    ).await {
6729                        return;
6730                    }
6731                    // Reconcile the pending respawn with what the command did:
6732                    // a start may already have spawned a fresh child,
6733                    // while a disable or drain moved the snapshot out of the
6734                    // state the respawn was counting down from.
6735                    if child.is_some() || !respawn_still_pending(&snapshot) {
6736                        pending_respawn = None;
6737                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6738                    }
6739                }
6740            }
6741        } else {
6742            let command = if lock_snapshot(&snapshot)
6743                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6744            {
6745                tokio::select! {
6746                    command = commands.recv() => command,
6747                    _ = previous_instance_recheck.tick() => {
6748                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6749                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6750                            if enabled && !runtime.child_roster.is_closed() {
6751                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6752                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6753                                    Ok(next_child) => child = Some(next_child),
6754                                    Err(error) => {
6755                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6756                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6757                                    }
6758                                }
6759                            } else {
6760                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6761                                    state.waiting_for_previous_pid = None;
6762                                    state.waiting_for_previous_detail = None;
6763                                });
6764                            }
6765                        }
6766                        continue;
6767                    }
6768                }
6769            } else {
6770                commands.recv().await
6771            };
6772            let Some(command) = command else {
6773                return;
6774            };
6775            if !handle_supervisor_command(
6776                command,
6777                &mut spec,
6778                &mut runtime,
6779                &registry,
6780                &process_liveness,
6781                &snapshot,
6782                &mut child,
6783                &mut commands,
6784                &mut requeued,
6785            )
6786            .await
6787            {
6788                return;
6789            }
6790        }
6791    }
6792}
6793
6794fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6795    info!(
6796        module_id,
6797        restart_in_window = schedule.restart_in_window,
6798        delay_ms = schedule.delay.as_millis() as u64,
6799        "respawning after crash"
6800    );
6801}
6802
6803/// Whether the respawn a backoff was counting down to is still wanted. A
6804/// disable or drain handled while the backoff elapsed moves the snapshot out
6805/// of `Restarting`, and the operator's stop must win over the pending respawn,
6806/// so every sleep-then-spawn path re-validates against the live snapshot
6807/// instead of assuming the state it left behind still holds.
6808fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6809    matches!(
6810        lock_snapshot(snapshot),
6811        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6812    )
6813}
6814
6815enum NextAction {
6816    Stop {
6817        registration_released: bool,
6818    },
6819    Restart {
6820        schedule: Option<CrashRestartSchedule>,
6821    },
6822}
6823
6824#[allow(clippy::too_many_arguments)]
6825async fn handle_supervisor_command(
6826    command: SupervisorCommand,
6827    spec: &mut ModuleSpec,
6828    runtime: &mut SupervisorRuntimeConfig,
6829    registry: &Arc<Registry>,
6830    process_liveness: &SupervisorProcessLiveness,
6831    snapshot: &SharedSnapshot,
6832    child: &mut Option<SupervisedChild>,
6833    commands: &mut mpsc::Receiver<SupervisorCommand>,
6834    requeued: &mut VecDeque<SupervisorCommand>,
6835) -> bool {
6836    let command =
6837        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6838            match command {
6839                SupervisorCommand::Restart { reply, .. }
6840                | SupervisorCommand::Reload { reply, .. }
6841                | SupervisorCommand::Swap { reply, .. } => {
6842                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6843                    return true;
6844                }
6845                SupervisorCommand::SetEnabled {
6846                    enabled: true,
6847                    reply,
6848                    ..
6849                } => {
6850                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6851                    return true;
6852                }
6853                other => other,
6854            }
6855        } else {
6856            command
6857        };
6858    match command {
6859        SupervisorCommand::Drain { requester, reply } => {
6860            // A plain stop runs no forwarding drain, so nothing reaches the
6861            // module over its connection before the wait: ask by signal.
6862            let result = drain_optional_child(
6863                &spec.module_id,
6864                spec.protocol,
6865                StopNotice::NotSent,
6866                registry,
6867                runtime.forwarding.as_deref(),
6868                snapshot,
6869                &runtime.terminal_ring,
6870                &runtime.spawn_events,
6871                child,
6872                runtime.drain_timeout,
6873                ModuleState::Stopped,
6874                None,
6875            )
6876            .await;
6877            let registration_released = result.is_ok();
6878            if registration_released {
6879                runtime.record_activity_transition(
6880                    "subc.module_stopped",
6881                    &spec.module_id,
6882                    requester,
6883                );
6884            }
6885            let _ = reply.send(result);
6886            if registration_released {
6887                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6888            }
6889            false
6890        }
6891        SupervisorCommand::Retire { requester, reply } => {
6892            let result = async {
6893                let stop_notice = begin_forwarding_drain_if_configured(
6894                    spec,
6895                    runtime,
6896                    registry,
6897                    snapshot,
6898                    None,
6899                    RouteCloseReason::Disable,
6900                )
6901                .await?;
6902                drain_optional_child(
6903                    &spec.module_id,
6904                    spec.protocol,
6905                    stop_notice,
6906                    registry,
6907                    runtime.forwarding.as_deref(),
6908                    snapshot,
6909                    &runtime.terminal_ring,
6910                    &runtime.spawn_events,
6911                    child,
6912                    runtime.drain_timeout,
6913                    ModuleState::Stopped,
6914                    None,
6915                )
6916                .await
6917            }
6918            .await;
6919            let registration_released = result.is_ok();
6920            if registration_released {
6921                runtime.record_activity_transition(
6922                    "subc.module_stopped",
6923                    &spec.module_id,
6924                    requester,
6925                );
6926            }
6927            let _ = reply.send(result);
6928            if registration_released {
6929                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6930            }
6931            false
6932        }
6933        SupervisorCommand::Restart {
6934            requester,
6935            drain_timeout_ms,
6936            received_at_generation,
6937            queued_at,
6938            reply,
6939        } => {
6940            // Without this line a restart that waited in the queue (behind a
6941            // health probe cycle or another command) was invisible: the log
6942            // showed only the drain timing out, minutes after the operator's call.
6943            info!(
6944                module_id = %spec.module_id,
6945                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6946                "restart command dequeued"
6947            );
6948            // ACK AT INITIATION, not completion. The blocking form deadlocked any
6949            // caller whose own request lane rides the module being restarted: the
6950            // caller's in-flight request keeps the drain from quiescing, the drain
6951            // keeps the restart from completing, and the completion keeps the reply
6952            // from releasing the caller — so the drain always timed out and cut the
6953            // initiator with a GOODBYE, even on a healthy module. Replying once the
6954            // restart is validated lets a self-lane caller settle, which is exactly
6955            // what makes the drain succeed. Completion is observable via
6956            // supervisor.list / module status; a post-ack failure lands the module
6957            // in a visible terminal state below rather than in a reply nobody can
6958            // receive.
6959            let validation = match lock_snapshot(snapshot) {
6960                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6961                    module_id: spec.module_id.clone(),
6962                }),
6963                Ok(_) => Ok(()),
6964                Err(err) => Err(err),
6965            };
6966            let initiated = validation.is_ok();
6967            let _ = reply.send(validation);
6968            // A restart asks for a fresh process. Commands run one at a time,
6969            // so a restart queued behind another restart (two operator calls
6970            // in quick succession) is dequeued the moment the first one has
6971            // spawned its replacement -- before that process has sent HELLO.
6972            // Running it would drain and kill the process the first restart
6973            // just produced, which is the opposite of what both callers asked
6974            // for. If a process spawned after this request was received is
6975            // still supervised, the request is already satisfied. Not when the
6976            // configuration changed since that spawn: then the newer process
6977            // predates the spec this restart may exist to apply.
6978            // Generation publication can lag a child that is already serving
6979            // while its identity is recorded. Only process creation after request
6980            // receipt satisfies a restart, not publication after receipt.
6981            let satisfied_by_generation = if initiated
6982                && child
6983                    .as_ref()
6984                    .is_some_and(|child| child.spawned_at > queued_at)
6985            {
6986                lock_snapshot(snapshot).ok().and_then(|state| {
6987                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
6988                })
6989            } else {
6990                None
6991            };
6992            let satisfied_by_pending = initiated
6993                && child.is_none()
6994                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6995                    // No child exists yet, so this replacement will necessarily
6996                    // be created after the queued request, unlike a published
6997                    // generation whose process may already have been serving.
6998                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6999                    if pending {
7000                        state.coalesced_restart_pending = true;
7001                    }
7002                    pending
7003                });
7004            if satisfied_by_pending {
7005                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7006            } else if let Some(generation) = satisfied_by_generation {
7007                info!(
7008                    module_id = %spec.module_id,
7009                    received_at_generation,
7010                    "restart already satisfied by generation {generation}; not restarting again"
7011                );
7012            } else if initiated {
7013                // Precedence: this restart's operator override, else the module's
7014                // configured budget (already resolved into the runtime).
7015                let drain_timeout = drain_timeout_ms
7016                    .map(Duration::from_millis)
7017                    .unwrap_or(runtime.drain_timeout);
7018                if let Err(err) = restart_child(
7019                    spec,
7020                    runtime,
7021                    registry,
7022                    process_liveness,
7023                    snapshot,
7024                    child,
7025                    drain_timeout,
7026                )
7027                .await
7028                {
7029                    warn!(
7030                        module_id = %spec.module_id,
7031                        error = %err,
7032                        "operator restart failed after initiation ack; module state carries the outcome"
7033                    );
7034                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7035                        state.state = ModuleState::Failed;
7036                        clear_current_process_facts(state);
7037                    });
7038                } else {
7039                    runtime.start_requester = requester;
7040                    runtime.pending_activity_restart = true;
7041                }
7042            }
7043            true
7044        }
7045        SupervisorCommand::Reload { requester, reply } => {
7046            let result =
7047                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7048            if result.is_ok() {
7049                runtime.start_requester = requester;
7050                runtime.pending_activity_restart = true;
7051            }
7052            if result.is_ok()
7053                && runtime
7054                    .scheduled_respawn
7055                    .lock()
7056                    .unwrap_or_else(|p| p.into_inner())
7057                    .as_ref()
7058                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7059            {
7060                *runtime
7061                    .deferred_reload_reply
7062                    .lock()
7063                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7064            } else {
7065                let _ = reply.send(result);
7066            }
7067            true
7068        }
7069        SupervisorCommand::SetEnabled {
7070            enabled,
7071            requester,
7072            reply,
7073        } => {
7074            let previous_requester = runtime.start_requester.clone();
7075            if enabled {
7076                runtime.start_requester = requester.clone();
7077            }
7078            let result = set_child_enabled(
7079                spec,
7080                runtime,
7081                registry,
7082                process_liveness,
7083                snapshot,
7084                child,
7085                enabled,
7086            )
7087            .await;
7088            if matches!(result, Ok(true)) {
7089                runtime.pending_activity_restart = false;
7090                if enabled {
7091                    runtime.start_requester = requester;
7092                } else {
7093                    runtime.start_requester = None;
7094                    runtime.record_activity_transition(
7095                        "subc.module_disabled",
7096                        &spec.module_id,
7097                        requester,
7098                    );
7099                }
7100            } else {
7101                runtime.start_requester = previous_requester;
7102            }
7103            let _ = reply.send(result);
7104            true
7105        }
7106        SupervisorCommand::UpdateConfiguration {
7107            spec: next_spec,
7108            health,
7109            drain_timeout_ms,
7110            reply,
7111        } => {
7112            if let Some(handle) = &runtime.supervisor_handle {
7113                handle.apply_identity_configuration(&next_spec);
7114            }
7115            *spec = next_spec;
7116            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7117                state.configuration_updated_since_spawn = true;
7118            });
7119            let health_changed = runtime.health != health;
7120            runtime.health = health;
7121            // Reset the cadence and old endpoint's failure streak on a live
7122            // health-policy change rather than waiting for its old deadline.
7123            if health_changed {
7124                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7125                    state.health = ModuleHealthStatus::default();
7126                });
7127            }
7128            runtime.drain_timeout = drain_timeout_ms
7129                .map(Duration::from_millis)
7130                .unwrap_or(runtime.default_drain_timeout);
7131            *runtime
7132                .effective_drain_timeout
7133                .lock()
7134                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7135            let _ = reply.send(());
7136            true
7137        }
7138        SupervisorCommand::Swap {
7139            requester,
7140            ready_timeout,
7141            reply,
7142        } => {
7143            let mut swap_runtime = runtime.clone();
7144            swap_runtime.start_requester = requester;
7145            let before = lock_snapshot(snapshot)
7146                .ok()
7147                .map(|state| state.spawn_generation);
7148            let end = swap::run_swap(
7149                spec,
7150                &swap_runtime,
7151                registry,
7152                process_liveness,
7153                snapshot,
7154                child,
7155                commands,
7156                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7157                reply,
7158            )
7159            .await;
7160            if before
7161                != lock_snapshot(snapshot)
7162                    .ok()
7163                    .map(|state| state.spawn_generation)
7164            {
7165                runtime.start_requester = swap_runtime.start_requester;
7166            }
7167            requeued.extend(end.requeue);
7168            true
7169        }
7170    }
7171}
7172
7173async fn restart_child(
7174    spec: &ModuleSpec,
7175    runtime: &SupervisorRuntimeConfig,
7176    registry: &Registry,
7177    process_liveness: &SupervisorProcessLiveness,
7178    snapshot: &SharedSnapshot,
7179    child: &mut Option<SupervisedChild>,
7180    drain_timeout: Duration,
7181) -> Result<(), SuperviseError> {
7182    // Restart cycles a running module; it must not silently start a disabled one.
7183    if !lock_snapshot(snapshot)?.enabled {
7184        return Err(SuperviseError::Disabled {
7185            module_id: spec.module_id.clone(),
7186        });
7187    }
7188    let stop_notice = begin_forwarding_drain_with_timeout(
7189        spec,
7190        runtime,
7191        registry,
7192        snapshot,
7193        None,
7194        RouteCloseReason::Restart,
7195        drain_timeout,
7196    )
7197    .await?;
7198
7199    if child.is_some() {
7200        drain_optional_child(
7201            &spec.module_id,
7202            spec.protocol,
7203            stop_notice,
7204            registry,
7205            runtime.forwarding.as_deref(),
7206            snapshot,
7207            &runtime.terminal_ring,
7208            &runtime.spawn_events,
7209            child,
7210            drain_timeout,
7211            ModuleState::Restarting,
7212            Some(true),
7213        )
7214        .await?;
7215    } else {
7216        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7217            state.enabled = true;
7218            state.state = ModuleState::Restarting;
7219            clear_current_process_facts(state);
7220        })?;
7221        release_dead_registration(
7222            registry,
7223            runtime.forwarding.as_deref(),
7224            snapshot,
7225            &spec.module_id,
7226        )
7227        .await?;
7228    }
7229
7230    reset_restart_count(snapshot, &spec.module_id)?;
7231    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7232    schedule_respawn(
7233        runtime,
7234        snapshot,
7235        &spec.module_id,
7236        runtime.restart_policy.backoff,
7237        RespawnKind::Spawn,
7238    )
7239}
7240
7241async fn reload_child(
7242    spec: &ModuleSpec,
7243    runtime: &SupervisorRuntimeConfig,
7244    registry: &Registry,
7245    process_liveness: &SupervisorProcessLiveness,
7246    snapshot: &SharedSnapshot,
7247    child: &mut Option<SupervisedChild>,
7248) -> Result<(), SuperviseError> {
7249    // Reload cycles a running module; it must not silently start a disabled one.
7250    if !lock_snapshot(snapshot)?.enabled {
7251        return Err(SuperviseError::Disabled {
7252            module_id: spec.module_id.clone(),
7253        });
7254    }
7255    let stop_notice = begin_forwarding_drain(
7256        spec,
7257        runtime,
7258        registry,
7259        snapshot,
7260        Some(true),
7261        RouteCloseReason::Reload,
7262    )
7263    .await?;
7264
7265    if child.is_some() {
7266        drain_optional_child(
7267            &spec.module_id,
7268            spec.protocol,
7269            stop_notice,
7270            registry,
7271            runtime.forwarding.as_deref(),
7272            snapshot,
7273            &runtime.terminal_ring,
7274            &runtime.spawn_events,
7275            child,
7276            runtime.drain_timeout,
7277            ModuleState::Restarting,
7278            Some(true),
7279        )
7280        .await?;
7281    } else {
7282        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7283            state.enabled = true;
7284            state.state = ModuleState::Restarting;
7285            clear_current_process_facts(state);
7286        })?;
7287        release_dead_registration(
7288            registry,
7289            runtime.forwarding.as_deref(),
7290            snapshot,
7291            &spec.module_id,
7292        )
7293        .await?;
7294    }
7295
7296    reset_restart_count(snapshot, &spec.module_id)?;
7297    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7298    schedule_respawn(
7299        runtime,
7300        snapshot,
7301        &spec.module_id,
7302        runtime.restart_policy.backoff,
7303        RespawnKind::Reload,
7304    )
7305}
7306
7307async fn finish_reload_child(
7308    spec: &ModuleSpec,
7309    runtime: &SupervisorRuntimeConfig,
7310    registry: &Registry,
7311    process_liveness: &SupervisorProcessLiveness,
7312    snapshot: &SharedSnapshot,
7313    child: &mut Option<SupervisedChild>,
7314) -> Result<(), SuperviseError> {
7315    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7316    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7317        Ok(next_child) => next_child,
7318        Err(err) => {
7319            return handle_reload_spawn_failure(
7320                spec,
7321                runtime,
7322                process_liveness,
7323                snapshot,
7324                child,
7325                format!("new child failed to spawn: {err}"),
7326            )
7327            .await;
7328        }
7329    };
7330    *child = Some(next_child);
7331
7332    let wait_outcome = {
7333        let active_child = child.as_mut().expect("new reload child was just stored");
7334        wait_for_registration_after_reload(
7335            registry,
7336            &spec.module_id,
7337            snapshot,
7338            active_child,
7339            REGISTRY_RELEASE_TIMEOUT,
7340        )
7341        .await?
7342    };
7343
7344    match wait_outcome {
7345        RegistrationWaitOutcome::Registered => {
7346            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7347            Ok(())
7348        }
7349        RegistrationWaitOutcome::Exited(exit_report) => {
7350            if let Some(active_child) = child.as_mut() {
7351                active_child.drain_stderr(&spec.module_id).await;
7352            }
7353            // Keep the reaped child's roster guard until its terminal is written.
7354            // Shutdown waits on that guard, not on the child Option used for respawn.
7355            let mut exited_child = child.take().expect("exited reload child is still stored");
7356            #[cfg(test)]
7357            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7358                gate.reached.notify_one();
7359                gate.resume.notified().await;
7360            }
7361            let result = handle_reload_child_registration_failure(
7362                spec,
7363                runtime,
7364                registry,
7365                process_liveness,
7366                snapshot,
7367                child,
7368                ReloadRegistrationFailure {
7369                    exit_report: registration_failure_exit_report(exit_report),
7370                    reason: exited_child
7371                        .spawn_failure
7372                        .clone()
7373                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7374                },
7375            )
7376            .await;
7377            exited_child.release_roster();
7378            result
7379        }
7380        RegistrationWaitOutcome::TimedOut => {
7381            let mut timed_out_child = child
7382                .take()
7383                .expect("timed-out reload child is still running");
7384            timed_out_child
7385                .start_kill()
7386                .map_err(|source| SuperviseError::Kill {
7387                    module_id: spec.module_id.clone(),
7388                    source,
7389                })?;
7390            let status = timed_out_child
7391                .wait()
7392                .await
7393                .map_err(|source| SuperviseError::Wait {
7394                    module_id: spec.module_id.clone(),
7395                    source,
7396                })?;
7397            timed_out_child.drain_stderr(&spec.module_id).await;
7398            handle_reload_child_registration_failure(
7399                spec,
7400                runtime,
7401                registry,
7402                process_liveness,
7403                snapshot,
7404                child,
7405                ReloadRegistrationFailure {
7406                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7407                        snapshot,
7408                        &timed_out_child,
7409                        &status,
7410                    )),
7411                    reason: format!(
7412                        "new child did not register within {:?}",
7413                        REGISTRY_RELEASE_TIMEOUT
7414                    ),
7415                },
7416            )
7417            .await
7418        }
7419    }
7420}
7421
7422async fn set_child_enabled(
7423    spec: &ModuleSpec,
7424    runtime: &SupervisorRuntimeConfig,
7425    registry: &Registry,
7426    process_liveness: &SupervisorProcessLiveness,
7427    snapshot: &SharedSnapshot,
7428    child: &mut Option<SupervisedChild>,
7429    enabled: bool,
7430) -> Result<bool, SuperviseError> {
7431    let (current_enabled, current_state, respawn_pending) = {
7432        let state = lock_snapshot(snapshot)?;
7433        (state.enabled, state.state, state.respawn_pending)
7434    };
7435    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7436    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7437    // clean (Stopped) has no live process and no other in-band recovery — the
7438    // operator's start is the explicit recovery act and resets the budget. Without
7439    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7440    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7441    // the one providing every agent's shell.
7442    let revive_terminal = enabled
7443        && current_enabled
7444        && child.is_none()
7445        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7446            || (current_state == ModuleState::Restarting && !respawn_pending));
7447    if current_enabled == enabled && !revive_terminal {
7448        return Ok(false);
7449    }
7450
7451    if enabled {
7452        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7453            state.enabled = true;
7454            state.state = ModuleState::Starting;
7455            clear_current_process_facts(state);
7456        })?;
7457        #[cfg(test)]
7458        if runtime.test_seed_stale_facts_before_enable_spawn {
7459            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7460                state.process_alive = true;
7461                state.pid = Some(41);
7462                state.spawned_at_ms = Some(42);
7463                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7464                state.spawned_file_identity = Some(SpawnedFileIdentity {
7465                    device: 43,
7466                    inode: 44,
7467                    #[cfg(windows)]
7468                    windows_image: None,
7469                });
7470            })?;
7471        }
7472        release_dead_registration(
7473            registry,
7474            runtime.forwarding.as_deref(),
7475            snapshot,
7476            &spec.module_id,
7477        )
7478        .await?;
7479        reset_restart_count(snapshot, &spec.module_id)?;
7480        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7481        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7482            Ok(next_child) => next_child,
7483            Err(err) => {
7484                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7485                    state.state = ModuleState::Failed;
7486                    clear_current_process_facts(state);
7487                }) {
7488                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7489                }
7490                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7491                return Err(err);
7492            }
7493        };
7494        *child = Some(next_child);
7495        debug!(module_id = %spec.module_id, "supervised module enabled");
7496        Ok(true)
7497    } else {
7498        let stop_notice = begin_forwarding_drain_if_configured(
7499            spec,
7500            runtime,
7501            registry,
7502            snapshot,
7503            Some(false),
7504            RouteCloseReason::Disable,
7505        )
7506        .await?;
7507        drain_optional_child(
7508            &spec.module_id,
7509            spec.protocol,
7510            stop_notice,
7511            registry,
7512            runtime.forwarding.as_deref(),
7513            snapshot,
7514            &runtime.terminal_ring,
7515            &runtime.spawn_events,
7516            child,
7517            runtime.drain_timeout,
7518            ModuleState::Disabled,
7519            Some(false),
7520        )
7521        .await?;
7522        debug!(module_id = %spec.module_id, "supervised module disabled");
7523        Ok(true)
7524    }
7525}
7526
7527#[allow(clippy::too_many_arguments)]
7528async fn on_child_exit(
7529    spec: &ModuleSpec,
7530    policy: RestartPolicy,
7531    registry: &Registry,
7532    snapshot: &SharedSnapshot,
7533    terminal_ring: &Arc<Mutex<TerminalRing>>,
7534    spawn_events: &SpawnEventFeed,
7535    roster: &ChildRoster,
7536    exit_report: ExitReport,
7537) -> NextAction {
7538    // Once the daemon has begun shutting down, no exit is a crash to recover
7539    // from: the module is exiting because the daemon is going away (EOF on its
7540    // connection, or a service manager signalling the whole cgroup). Record it
7541    // as such and never schedule a respawn, which would only start a process
7542    // for the shutdown to end again.
7543    if roster.is_closed() {
7544        return on_child_exit_during_daemon_shutdown(
7545            spec,
7546            registry,
7547            snapshot,
7548            terminal_ring,
7549            spawn_events,
7550            exit_report,
7551        )
7552        .await;
7553    }
7554    // Every stop the supervisor itself asks for (operator stop, disable,
7555    // restart, reload, swap, a health restart, a drain that runs out of budget)
7556    // takes the child out of the supervise loop and reaps it in
7557    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7558    // that reaches this point was not requested by the daemon.
7559    //
7560    // For a subc-wire module a clean exit is still a stop: those modules are
7561    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7562    // crash, and exiting 0 is a deliberate choice the module made. A
7563    // `protocol: "none"` module is a stock program we cannot change, and many
7564    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7565    // stop would leave the module down for good after any stray signal, so it
7566    // goes through the crash path instead: it spends restart budget, respawns
7567    // with the crash backoff, and ends `failed` when the budget runs out.
7568    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7569        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7570    match exit_report.kind {
7571        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7572            info!(
7573                module_id = %spec.module_id,
7574                exit_code = ?exit_report.code,
7575                exit_signal = ?exit_report.signal,
7576                "supervised module exited cleanly"
7577            );
7578            if let Err(err) = publish_after_terminal_record(
7579                &spec.module_id,
7580                || {
7581                    record_terminal(
7582                        &spec.module_id,
7583                        terminal_ring,
7584                        spawn_events,
7585                        &exit_report,
7586                        TerminalDisposition::Stopped,
7587                    )
7588                },
7589                || {
7590                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7591                        state.state = ModuleState::Stopped;
7592                        clear_current_process_facts(state);
7593                        state.last_exit = Some(exit_report.clone());
7594                    })
7595                },
7596            ) {
7597                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7598            }
7599            let registration_released = match wait_for_registration_release(
7600                registry,
7601                &spec.module_id,
7602                REGISTRY_RELEASE_TIMEOUT,
7603            )
7604            .await
7605            {
7606                Ok(()) => true,
7607                Err(err) => {
7608                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7609                    false
7610                }
7611            };
7612            NextAction::Stop {
7613                registration_released,
7614            }
7615        }
7616        ExitKind::Clean | ExitKind::Crash => {
7617            if unrequested_clean_exit_of_protocol_none {
7618                warn!(
7619                    module_id = %spec.module_id,
7620                    exit_code = ?exit_report.code,
7621                    exit_signal = ?exit_report.signal,
7622                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7623                );
7624            } else {
7625                warn!(
7626                    module_id = %spec.module_id,
7627                    exit_code = ?exit_report.code,
7628                    exit_signal = ?exit_report.signal,
7629                    "supervised module exited abnormally (crash)"
7630                );
7631            }
7632            let mut restart_schedule = None;
7633            let mut disposition = TerminalDisposition::Disabled;
7634            let mut final_state = ModuleState::Disabled;
7635            // Set only when the budget is what stopped the module, so the
7636            // terminal record says which limit was hit rather than leaving
7637            // `failed` to be read as "crashed once, badly".
7638            let mut disposition_detail = lock_snapshot(snapshot)
7639                .ok()
7640                .and_then(|mut state| state.spawn_failure.take());
7641            let now = Instant::now();
7642            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7643                if state.enabled {
7644                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7645                        // Reserving the last slot exhausts the budget for the
7646                        // replacement, not for this exit. Publish its scheduled
7647                        // recovery atomically with the reservation so connection
7648                        // cleanup sees it while terminal recording still holds
7649                        // the visible state at Running.
7650                        state.respawn_pending = true;
7651                        final_state = ModuleState::Restarting;
7652                        restart_schedule = Some(schedule);
7653                        disposition = TerminalDisposition::Restarting;
7654                    } else {
7655                        final_state = ModuleState::Failed;
7656                        disposition = TerminalDisposition::Failed;
7657                        let budget = policy.budget_exhausted_detail();
7658                        disposition_detail =
7659                            Some(disposition_detail.take().map_or_else(
7660                                || budget.clone(),
7661                                |cause| format!("{cause}; {budget}"),
7662                            ));
7663                    }
7664                } else {
7665                    disposition = TerminalDisposition::Disabled;
7666                }
7667            }) {
7668                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7669                return NextAction::Stop {
7670                    registration_released: false,
7671                };
7672            }
7673            if disposition == TerminalDisposition::Failed {
7674                // The window is in the message, not only in the fields: this line
7675                // is read in a scrollback where a bare `max_restarts=3` reads as a
7676                // lifetime cap and sends the operator looking for three crashes
7677                // that never happened together.
7678                error!(
7679                    module_id = %spec.module_id,
7680                    max_restarts = policy.max_restarts,
7681                    window_secs = policy.window.as_secs(),
7682                    "module stopped: {}",
7683                    policy.budget_exhausted_detail()
7684                );
7685            }
7686            if let Err(err) = publish_after_terminal_record(
7687                &spec.module_id,
7688                || {
7689                    record_terminal_with_detail(
7690                        &spec.module_id,
7691                        terminal_ring,
7692                        spawn_events,
7693                        &exit_report,
7694                        disposition,
7695                        disposition_detail,
7696                    )
7697                },
7698                || {
7699                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7700                        state.state = final_state;
7701                        clear_current_process_facts(state);
7702                        state.last_exit = Some(exit_report.clone());
7703                    })
7704                },
7705            ) {
7706                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7707            }
7708
7709            if let Some(schedule) = restart_schedule {
7710                NextAction::Restart {
7711                    schedule: Some(schedule),
7712                }
7713            } else {
7714                let registration_released = match wait_for_registration_release(
7715                    registry,
7716                    &spec.module_id,
7717                    REGISTRY_RELEASE_TIMEOUT,
7718                )
7719                .await
7720                {
7721                    Ok(()) => true,
7722                    Err(err) => {
7723                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7724                        false
7725                    }
7726                };
7727                NextAction::Stop {
7728                    registration_released,
7729                }
7730            }
7731        }
7732        ExitKind::DeliberateSeverance => {
7733            warn!(
7734                module_id = %spec.module_id,
7735                exit_code = ?exit_report.code,
7736                exit_signal = ?exit_report.signal,
7737                "supervised module exited after deliberate connection severance"
7738            );
7739            let should_restart = match lock_snapshot(snapshot) {
7740                Ok(state) => state.enabled,
7741                Err(err) => {
7742                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7743                    return NextAction::Stop {
7744                        registration_released: false,
7745                    };
7746                }
7747            };
7748            let final_state = if should_restart {
7749                ModuleState::Restarting
7750            } else {
7751                ModuleState::Disabled
7752            };
7753            if let Err(err) = publish_after_terminal_record(
7754                &spec.module_id,
7755                || {
7756                    record_terminal(
7757                        &spec.module_id,
7758                        terminal_ring,
7759                        spawn_events,
7760                        &exit_report,
7761                        terminal_disposition(final_state),
7762                    )
7763                },
7764                || {
7765                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7766                        state.state = final_state;
7767                        clear_current_process_facts(state);
7768                        state.last_exit = Some(exit_report.clone());
7769                        state.lifetime_restarts += 1;
7770                    })
7771                },
7772            ) {
7773                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7774                return NextAction::Stop {
7775                    registration_released: false,
7776                };
7777            }
7778
7779            if should_restart {
7780                NextAction::Restart { schedule: None }
7781            } else {
7782                let registration_released = match wait_for_registration_release(
7783                    registry,
7784                    &spec.module_id,
7785                    REGISTRY_RELEASE_TIMEOUT,
7786                )
7787                .await
7788                {
7789                    Ok(()) => true,
7790                    Err(err) => {
7791                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7792                        false
7793                    }
7794                };
7795                NextAction::Stop {
7796                    registration_released,
7797                }
7798            }
7799        }
7800    }
7801}
7802
7803async fn on_child_exit_during_daemon_shutdown(
7804    spec: &ModuleSpec,
7805    registry: &Registry,
7806    snapshot: &SharedSnapshot,
7807    terminal_ring: &Arc<Mutex<TerminalRing>>,
7808    spawn_events: &SpawnEventFeed,
7809    exit_report: ExitReport,
7810) -> NextAction {
7811    info!(
7812        module_id = %spec.module_id,
7813        exit_code = ?exit_report.code,
7814        exit_signal = ?exit_report.signal,
7815        exit_kind = ?exit_report.kind,
7816        "supervised module exited during daemon shutdown; not restarting it"
7817    );
7818    if let Err(err) = publish_after_terminal_record(
7819        &spec.module_id,
7820        || {
7821            record_terminal(
7822                &spec.module_id,
7823                terminal_ring,
7824                spawn_events,
7825                &exit_report,
7826                TerminalDisposition::DaemonShutdown,
7827            )
7828        },
7829        || {
7830            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7831                state.state = ModuleState::Stopped;
7832                clear_current_process_facts(state);
7833                state.last_exit = Some(exit_report.clone());
7834            })
7835        },
7836    ) {
7837        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7838    }
7839    let registration_released =
7840        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7841            .await
7842            .is_ok();
7843    NextAction::Stop {
7844        registration_released,
7845    }
7846}
7847
7848// Keep the status lock out of terminal I/O. Readers may observe the previous
7849// state while recording is in progress, but never a completed exit without its
7850// history. Journal errors are logged by the journal and retained in the ring;
7851// a recording panic still publishes the final state before resuming the unwind.
7852fn publish_after_terminal_record<T>(
7853    module_id: &str,
7854    record: impl FnOnce(),
7855    publish: impl FnOnce() -> T,
7856) -> T {
7857    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7858    let published = publish();
7859    if let Err(panic) = recorded {
7860        error!(
7861            module_id,
7862            "terminal recording panicked; published final state before resuming unwind"
7863        );
7864        std::panic::resume_unwind(panic);
7865    }
7866    published
7867}
7868
7869fn fail_after_wait_error(
7870    module_id: &str,
7871    snapshot: &SharedSnapshot,
7872    terminal_ring: &Arc<Mutex<TerminalRing>>,
7873    spawn_events: &SpawnEventFeed,
7874    detail: String,
7875) {
7876    let report = wait_error_exit_report();
7877    publish_after_terminal_record(
7878        module_id,
7879        || {
7880            record_terminal_with_detail(
7881                module_id,
7882                terminal_ring,
7883                spawn_events,
7884                &report,
7885                TerminalDisposition::Failed,
7886                Some(detail),
7887            )
7888        },
7889        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7890    );
7891}
7892
7893#[cfg(test)]
7894fn record_wait_error_terminal(
7895    module_id: &str,
7896    terminal_ring: &Arc<Mutex<TerminalRing>>,
7897    spawn_events: &SpawnEventFeed,
7898) {
7899    record_terminal(
7900        module_id,
7901        terminal_ring,
7902        spawn_events,
7903        &wait_error_exit_report(),
7904        TerminalDisposition::Failed,
7905    );
7906}
7907
7908fn record_terminal(
7909    module_id: &str,
7910    terminal_ring: &Arc<Mutex<TerminalRing>>,
7911    spawn_events: &SpawnEventFeed,
7912    exit_report: &ExitReport,
7913    disposition: TerminalDisposition,
7914) {
7915    record_terminal_with_detail(
7916        module_id,
7917        terminal_ring,
7918        spawn_events,
7919        exit_report,
7920        disposition,
7921        None,
7922    );
7923}
7924
7925/// The ring lock is held only to capture the read (see
7926/// `TerminalJournal::capture_read`), so this module's exits keep recording
7927/// while the journal files are read. Blocking: it reads files.
7928fn durable_terminal_history_of(
7929    terminal_ring: &Mutex<TerminalRing>,
7930    module_id: &str,
7931) -> subc_control::TerminalHistory {
7932    let read = terminal_ring
7933        .lock()
7934        .unwrap_or_else(|p| p.into_inner())
7935        .capture_durable_history();
7936    read.read(module_id)
7937}
7938
7939fn record_terminal_with_detail(
7940    module_id: &str,
7941    terminal_ring: &Arc<Mutex<TerminalRing>>,
7942    spawn_events: &SpawnEventFeed,
7943    exit_report: &ExitReport,
7944    disposition: TerminalDisposition,
7945    disposition_detail: Option<String>,
7946) {
7947    #[cfg(test)]
7948    terminal_history_tests::before_terminal_record(spawn_events);
7949    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7950    let record = TerminalRecord {
7951        exit_code: exit_report.code,
7952        exit_signal: exit_report.signal,
7953        at_ms: exit_report.at_ms,
7954        disposition,
7955        exit_kind: exit_report.kind.into(),
7956        disposition_detail,
7957    };
7958    terminal_ring
7959        .lock()
7960        .unwrap_or_else(|poisoned| poisoned.into_inner())
7961        .record_exit(module_id, record);
7962}
7963
7964fn untrack_if_registration_released(
7965    process_liveness: &SupervisorProcessLiveness,
7966    registry: &Registry,
7967    module_id: &str,
7968    snapshot: &SharedSnapshot,
7969) {
7970    match registry.get_module(module_id) {
7971        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7972        Ok(Some(_)) => {}
7973        Err(err) => {
7974            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7975        }
7976    }
7977}
7978
7979/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
7980/// then apply the module's configured entries minus daemon-private capture keys.
7981///
7982/// Separated from `spawn_child` only so it can be asserted without spawning a
7983/// process — a duplicate of this logic in a test would pass while the real one
7984/// drifted, which is the defect class this function exists to avoid.
7985/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
7986/// nonce. A `protocol: "none"` module gets neither, because it cannot use
7987/// either and the argument would stop a stock binary from starting at all.
7988/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
7989///
7990/// The plain-spawn form, kept for the tests that assert its plan; spawns go
7991/// through [`apply_wire_spawn_args_for_role`].
7992#[cfg(test)]
7993fn apply_wire_spawn_args(
7994    command: &mut Command,
7995    spec: &ModuleSpec,
7996    connection_file_path: Option<&std::path::Path>,
7997    handle: Option<&SupervisorHandle>,
7998) -> Result<Option<NonceHandoff>, SuperviseError> {
7999    apply_wire_spawn_args_for_role(
8000        command,
8001        spec,
8002        connection_file_path,
8003        handle,
8004        SpawnRole::Plain,
8005    )
8006}
8007
8008/// The read end of a spawn's launch-nonce pipe, prepared by
8009/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8010/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8011/// server before spawn and authenticates its client after the child exists.
8012#[cfg(unix)]
8013type NonceHandoff = subc_os::LaunchNonceHandoff;
8014#[cfg(windows)]
8015type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8016#[cfg(not(any(unix, windows)))]
8017type NonceHandoff = std::convert::Infallible;
8018
8019/// Create the launch nonce that a new module process must present when it
8020/// registers or opens a consumer connection, including a replacement process
8021/// started alongside a still-serving module.
8022///
8023/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8024/// a separate candidate token so the still-serving incumbent and its consumers
8025/// keep their nonce. Both records are installed before the process exists, so
8026/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8027///
8028/// On Unix the nonce is delivered only through a pipe. It is written into
8029/// a pipe whose read end the child gets as descriptor 3, named by
8030/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8031/// process of the same user cannot read it with `ps eww`. That handoff is
8032/// returned rather than installed here, because installing it replaces
8033/// whatever the child has at descriptor 3 and so must be the last pre-exec
8034/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8035/// the Linux callback that writes the process into its cgroup. Only after that
8036/// does `handoff.install_last` register the descriptor-3 replacement, so it
8037/// cannot close a descriptor the cgroup callback still needs.
8038/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8039fn apply_wire_spawn_args_for_role(
8040    command: &mut Command,
8041    spec: &ModuleSpec,
8042    connection_file_path: Option<&std::path::Path>,
8043    handle: Option<&SupervisorHandle>,
8044    role: SpawnRole,
8045) -> Result<Option<NonceHandoff>, SuperviseError> {
8046    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8047    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8048    // included: a daemon started from a module's process tree inherits it,
8049    // and passing it on would point the child at a descriptor it does not
8050    // have.
8051    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8052    #[cfg(windows)]
8053    {
8054        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8055        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8056    }
8057    // Remove inherited or configured copies too: withholding must mean absent.
8058    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8059    if spec.protocol == ModuleProtocol::None {
8060        return Ok(None);
8061    }
8062    if let Some(connection_file_path) = connection_file_path {
8063        command.arg(SUBC_ARG).arg(connection_file_path);
8064    }
8065
8066    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8067    // route.open attestation. Reserved modules additionally use the same nonce
8068    // for HELLO id-squatting protection. A respawn rotates both records.
8069    let nonce = generate_launch_nonce()?;
8070    if let Some(handle) = handle {
8071        match role {
8072            SpawnRole::Plain => {
8073                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8074                if spec.reserved {
8075                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8076                }
8077            }
8078            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8079        }
8080    }
8081    #[cfg(unix)]
8082    let handoff = {
8083        let handoff =
8084            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8085                program: spec.program.clone(),
8086                source,
8087                cgroup_path: None,
8088            })?;
8089        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8090        Some(handoff)
8091    };
8092    #[cfg(windows)]
8093    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8094    #[cfg(not(any(unix, windows)))]
8095    let handoff = None;
8096    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8097    // until the daemon's per-spawn delivery records show that every running
8098    // Windows module has read its secret from the named pipe.
8099    #[cfg(not(unix))]
8100    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8101    Ok(handoff)
8102}
8103
8104fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8105    command.env_remove(CK_LOG_ENV);
8106    // The spawn role is the supervisor's to set, and only on a swap candidate
8107    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8108    // it, is what makes it absent on a plain spawn: the daemon's own
8109    // environment could carry it, and so could a spec built outside daemon
8110    // config (config refuses it as an `env` key). A module reading it on a
8111    // plain restart would pick the long swap budget and leave callers waiting.
8112    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8113    for (key, value) in &spec.env {
8114        // cortexkit-log currently exposes retention only as a Rust struct, not
8115        // environment names. These values are daemon-private sink metadata and
8116        // must never become a public child-process contract by being inherited.
8117        if matches!(
8118            key.as_str(),
8119            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8120        ) || key == SUBC_SPAWN_ROLE_ENV
8121        {
8122            continue;
8123        }
8124        command.env(key, value);
8125    }
8126}
8127
8128#[cfg(windows)]
8129fn prepare_windows_nonce_pipe(
8130    command: &mut Command,
8131    spec: &ModuleSpec,
8132    nonce: &str,
8133) -> Option<NonceHandoff> {
8134    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8135        Ok(handoff) => {
8136            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8137            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8138            // still register until every module reads the pipe. A reader without
8139            // this variable set to env returns the pipe error instead of using
8140            // an inherited or configured environment copy.
8141            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8142            Some(handoff)
8143        }
8144        Err(error) => {
8145            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8146            None
8147        }
8148    }
8149}
8150
8151#[cfg(windows)]
8152fn start_windows_nonce_pipe(
8153    handoff: Option<NonceHandoff>,
8154    pid: u32,
8155    spec: &ModuleSpec,
8156) -> Option<subc_os::LaunchNoncePipeDelivery> {
8157    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8158    // same maximum wait used for registration during a reload. Older modules
8159    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8160    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8161        Ok(delivery) => Some(delivery),
8162        Err(error) => {
8163            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8164            None
8165        }
8166    })
8167}
8168
8169fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8170    #[cfg(windows)]
8171    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8172        return Some(
8173            if snapshot
8174                .pipe_consumed
8175                .as_ref()
8176                .is_some_and(|consumed| consumed.get().is_some())
8177            {
8178                "pipe"
8179            } else {
8180                "env"
8181            }
8182            .to_owned(),
8183        );
8184    }
8185    let _ = snapshot;
8186    None
8187}
8188
8189/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8190/// of a blue/green swap.
8191#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8192enum SpawnRole {
8193    Plain,
8194    SwapCandidate,
8195}
8196
8197/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8198/// `apply_child_env` has already removed the variable for every spawn.
8199fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8200    if role == SpawnRole::SwapCandidate {
8201        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8202    }
8203}
8204
8205fn previous_instance_error(
8206    spec: &ModuleSpec,
8207    pid: u32,
8208    reason: crate::live_children::RetainedReason,
8209) -> SuperviseError {
8210    SuperviseError::Spawn {
8211        program: spec.program.clone(),
8212        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8213        cgroup_path: None,
8214    }
8215}
8216
8217fn spawn_child(
8218    spec: &ModuleSpec,
8219    connection_file_path: Option<&std::path::Path>,
8220    handle: Option<&SupervisorHandle>,
8221    ring: &Arc<Mutex<StderrRing>>,
8222    capture_logs_dir: Option<&std::path::Path>,
8223    roster: &ChildRoster,
8224    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8225) -> Result<SupervisedChild, SuperviseError> {
8226    spawn_child_in_slot(
8227        spec,
8228        connection_file_path,
8229        handle,
8230        ring,
8231        capture_logs_dir,
8232        roster,
8233        #[cfg(target_os = "linux")]
8234        cgroup_placement,
8235        SpawnRole::Plain,
8236        false,
8237    )
8238}
8239
8240/// Spawn one process of `spec` into a slot.
8241///
8242/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8243/// A swap candidate needs a different cgroup from the process it is replacing,
8244/// which is still alive: in the same cgroup the two would be one kill domain,
8245/// and killing a failed candidate could take the incumbent with it.
8246///
8247/// The stderr capture file is `<module_id>.stderr.log` for every process of
8248/// the module, whichever slot it is in, because that is the one file
8249/// `ck module logs` reads. During a swap's overlap both processes append to it;
8250/// the daemon writes whole lines, so the two interleave by line, which is also
8251/// the merged view an operator wants while a swap runs.
8252#[allow(clippy::too_many_arguments)]
8253fn spawn_child_in_slot(
8254    spec: &ModuleSpec,
8255    connection_file_path: Option<&std::path::Path>,
8256    handle: Option<&SupervisorHandle>,
8257    ring: &Arc<Mutex<StderrRing>>,
8258    capture_logs_dir: Option<&std::path::Path>,
8259    roster: &ChildRoster,
8260    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8261    role: SpawnRole,
8262    alternate_slot: bool,
8263) -> Result<SupervisedChild, SuperviseError> {
8264    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8265        return Err(previous_instance_error(spec, pid, reason));
8266    }
8267    if roster.is_closed() {
8268        return Err(SuperviseError::Spawn {
8269            program: spec.program.clone(),
8270            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8271            cgroup_path: None,
8272        });
8273    }
8274    #[cfg(target_os = "linux")]
8275    let cgroup_name = {
8276        // Slot names alone are not kill domains: a retired incumbent may still
8277        // be draining when a later enable/restart spawns into the same slot.
8278        // Decimal entropy keeps the suffix unambiguous; Placement performs
8279        // the module-id escaping and constructs the filesystem path.
8280        if cgroup_placement.is_none() {
8281            swap::cgroup_name(&spec.module_id, alternate_slot)
8282        } else {
8283            let nonce = generate_launch_nonce()?;
8284            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8285            // Leave room for byte escaping and the suffix under NAME_MAX. The
8286            // label is only for humans; the nonce identifies the kill domain.
8287            let mut end = spec.module_id.len().min(64);
8288            while !spec.module_id.is_char_boundary(end) {
8289                end -= 1;
8290            }
8291            format!(
8292                "{}_{suffix}",
8293                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8294            )
8295        }
8296    };
8297    #[cfg(not(target_os = "linux"))]
8298    let _ = alternate_slot;
8299    #[cfg(target_os = "macos")]
8300    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8301    #[cfg(windows)]
8302    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8303        program: spec.program.clone(),
8304        source,
8305        cgroup_path: None,
8306    })?;
8307    #[cfg(windows)]
8308    let mut command = Command::new(image_capture.path());
8309    #[cfg(not(any(target_os = "macos", windows)))]
8310    let mut command = Command::new(&spec.program);
8311    command.args(&spec.args);
8312    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8313    // that is the whole of the intent, so remove that one key rather than the
8314    // environment.
8315    //
8316    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8317    // and took the POSIX environment with it. Modules spawned that way had no
8318    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8319    // logging:
8320    //
8321    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8322    //     both unset it fell back to the temp dir alone and `ck` could not find
8323    //     a daemon running on the same machine from inside any module's process
8324    //     tree — reporting a path the file has never lived at, which reads as
8325    //     "the daemon did not write its file".
8326    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8327    //     the RELATIVE `.local/share`, so a module deriving its own store path
8328    //     resolved it against its own CWD. That is the store-fragmentation
8329    //     defect the daemon already refuses in config (`parse_doc` rejects a
8330    //     relative `storage.data_home`) arriving by derivation instead.
8331    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8332    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8333    //     quietly rather than erroring.
8334    //
8335    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8336    // offered one candidate under /tmp while the file sat in /run/user/1000.
8337    //
8338    // A configured module is unaffected either way: `module_spec()` puts the
8339    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8340    // wins over anything ambient.
8341    apply_child_env(&mut command, spec);
8342    apply_spawn_role(&mut command, role);
8343    let nonce_handoff =
8344        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8345
8346    #[cfg(target_os = "linux")]
8347    let cgroup_path = cgroup_placement
8348        .map(|placement| placement.module_path(&cgroup_name))
8349        .transpose()
8350        .map_err(|source| SuperviseError::Cgroup {
8351            module_id: spec.module_id.clone(),
8352            source,
8353        })?;
8354    #[cfg(not(target_os = "linux"))]
8355    let cgroup_path: Option<PathBuf> = None;
8356    #[cfg(target_os = "linux")]
8357    if let Some(path) = &cgroup_path {
8358        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8359            if let Some(placement) = cgroup_placement {
8360                remove_module_cgroup(placement, &cgroup_name);
8361            }
8362            return Err(error);
8363        }
8364    }
8365
8366    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8367        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8368        match ChildOutputSink::open(&path, capture_retention(spec)) {
8369            Ok(sink) => sink,
8370            Err(error) => {
8371                warn!(
8372                    module_id = %spec.module_id,
8373                    path = %path.display(),
8374                    error = %error,
8375                    "could not open child output capture file; forwarding to stderr"
8376                );
8377                ChildOutputSink::Stderr
8378            }
8379        }
8380    } else {
8381        ChildOutputSink::Stderr
8382    };
8383
8384    command.stdout(Stdio::piped());
8385    command.stderr(Stdio::piped());
8386    command.kill_on_drop(true);
8387    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8388    // before exec). In the daemon's group, a service manager that kills the
8389    // job's process group when the daemon exits (launchd's default) killed
8390    // every module at the same moment its control connection closed, so no
8391    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8392    // module is reached only by the daemon: the EOF it sees when its
8393    // connection closes, and the bounded stop in `child_roster` for anything
8394    // still running after that. On Linux this composes with the cgroup
8395    // placement above: that is a pre_exec write to cgroup.procs, std performs
8396    // setpgid in the child before running pre_exec callbacks, and the two
8397    // change independent process attributes.
8398    //
8399    // stdin is /dev/null because a process outside the terminal's foreground
8400    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8401    // by hand would otherwise hand down. Under a service manager stdin is
8402    // already /dev/null.
8403    #[cfg(unix)]
8404    command.process_group(0);
8405    command.stdin(Stdio::null());
8406    // The LAST pre-exec step, after the cgroup placement above: installing the
8407    // nonce at descriptor 3 replaces whatever the child had there, which could
8408    // be the descriptor an earlier step writes through.
8409    #[cfg(unix)]
8410    if let Some(handoff) = nonce_handoff {
8411        handoff.install_last(command.as_std_mut());
8412    }
8413    #[cfg(not(any(unix, windows)))]
8414    let _ = nonce_handoff;
8415
8416    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8417    // cannot run a single instruction -- and therefore cannot spawn a
8418    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8419    // other two steps and why the window matters.
8420    #[cfg(windows)]
8421    {
8422        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8423            program: spec.program.clone(),
8424            source,
8425            cgroup_path: None,
8426        })?;
8427        subc_jobobject::suspend_on_create_async(&mut command);
8428    }
8429    // Timestamp before the spawn call: on Unix the child can serve before that
8430    // call even returns. This conservative bound never labels an older child
8431    // as a replacement for a request received during process creation.
8432    let spawned_at = Instant::now();
8433    let mut child = match command.spawn() {
8434        Ok(child) => child,
8435        Err(source) => {
8436            #[cfg(target_os = "linux")]
8437            if let Some(placement) = cgroup_placement {
8438                remove_module_cgroup(placement, &cgroup_name);
8439            }
8440            return Err(SuperviseError::Spawn {
8441                program: spec.program.clone(),
8442                source,
8443                cgroup_path,
8444            });
8445        }
8446    };
8447    #[cfg(test)]
8448    spawn_record_tests::after_create(&child, spec);
8449    // The parent must close its writer now: the acknowledgement pipe reports EOF
8450    // only when every writer is gone, and the child's copy closes when the
8451    // trampoline replaces itself with the module. Command holds only an integer
8452    // in its pre_exec callback, not another writer.
8453    #[cfg(target_os = "macos")]
8454    drop(exec_ack);
8455
8456    // Bind while the child is still suspended, before job assignment resumes it.
8457    // Keeping the executable handle open across CreateProcessW denies rename
8458    // and deletion, so the file cannot be exchanged during spawn. Bind that
8459    // file ID to the process handle returned by spawn, never a reopened PID.
8460    #[cfg(windows)]
8461    let windows_image = match image_capture.bind_tokio(&child) {
8462        Ok(image) => Arc::new(image),
8463        Err(source) => {
8464            kill_and_reap_rejected_child(&mut child);
8465            return Err(SuperviseError::Spawn {
8466                program: spec.program.clone(),
8467                source,
8468                cgroup_path: None,
8469            });
8470        }
8471    };
8472    // Assign the suspended child to its module job; resume only after its
8473    // process identity has been durably recorded for orphan recovery.
8474    #[cfg(windows)]
8475    let job = contain_spawned_child(&mut child, spec)?;
8476    let spawned_at_ms = unix_ms_now();
8477    #[cfg(not(windows))]
8478    let spawned_from = spec.program.clone();
8479    #[cfg(windows)]
8480    let spawned_from = windows_image.path().to_path_buf();
8481    #[cfg(not(windows))]
8482    let spawned_file_identity = spawned_file_identity(&spawned_from);
8483    #[cfg(windows)]
8484    let spawned_file_identity = Some(SpawnedFileIdentity {
8485        device: 0,
8486        inode: 0,
8487        windows_image: Some(Arc::clone(&windows_image)),
8488    });
8489    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8490        program: spec.program.clone(),
8491        source: io::Error::other("spawned child exposed no live pid"),
8492        cgroup_path: cgroup_path.clone(),
8493    })?;
8494    #[cfg(windows)]
8495    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8496    #[cfg(not(windows))]
8497    let process_start_time = crate::provenance::process_start_time(pid);
8498    #[cfg(windows)]
8499    let process_start_time = Some(windows_image.start_time());
8500    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8501    #[cfg(all(test, target_os = "macos"))]
8502    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8503    // Record the committed kernel image, not the configured pathname: PATH
8504    // lookup and shebang interpretation can select a different executable.
8505    // Linux's posix_spawn can return before that image is installed, so the
8506    // observer also waits for the kernel's first-exec flag to clear.
8507    #[cfg(not(windows))]
8508    let recorded_image = observe_spawned_image(pid);
8509    // spawn() confirms only the first exec, into the trampoline. Never persist
8510    // the trampoline image; the asynchronous acknowledgement publishes the
8511    // module image once the trampoline has replaced itself with the module.
8512    #[cfg(target_os = "macos")]
8513    let recorded_image = if privacy_exec.is_some() {
8514        None
8515    } else {
8516        recorded_image
8517    };
8518    #[cfg(target_os = "linux")]
8519    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8520    #[cfg(not(target_os = "linux"))]
8521    let recorded_cgroup_name = None;
8522    let roster_guard = roster.admit(
8523        spec.module_id.clone(),
8524        pid,
8525        spec.protocol,
8526        process_start_time,
8527        crate::child_roster::RecordedIdentity {
8528            #[cfg(not(windows))]
8529            start_time: recorded_image.map(|image| image.start_time),
8530            #[cfg(windows)]
8531            start_time: process_start_time,
8532            #[cfg(not(windows))]
8533            executable: recorded_image
8534                .and_then(|image| image.executable)
8535                .map(crate::live_children::ExecutableIdentity::from),
8536            #[cfg(windows)]
8537            executable: Some(crate::live_children::ExecutableIdentity::from(
8538                windows_image.identity(),
8539            )),
8540            cgroup_name: recorded_cgroup_name,
8541            #[cfg(target_os = "linux")]
8542            cgroup_placement: cgroup_placement.cloned(),
8543        },
8544    );
8545    let roster_guard = match roster_guard {
8546        Ok(guard) => guard,
8547        Err(source) => {
8548            #[cfg(windows)]
8549            if let Some(job) = &job {
8550                let _ = job.terminate();
8551            }
8552            #[cfg(target_os = "linux")]
8553            kill_module_cgroup(cgroup_placement, &cgroup_name);
8554            kill_and_reap_rejected_child(&mut child);
8555            #[cfg(target_os = "linux")]
8556            if let Some(placement) = cgroup_placement {
8557                remove_module_cgroup(placement, &cgroup_name);
8558            }
8559            return Err(SuperviseError::Spawn {
8560                program: spec.program.clone(),
8561                source: io::Error::other(format!(
8562                    "could not durably record module child: {source}"
8563                )),
8564                cgroup_path,
8565            });
8566        }
8567    };
8568    // The check at the top of this function can pass just before daemon
8569    // shutdown begins, and the process is only in the roster from here on.
8570    // The shutdown stop returns as soon as it finds the roster empty, so a
8571    // process admitted after that look would outlive the daemon. The roster
8572    // is closed before the stop first reads it and admission happens under
8573    // the roster's lock, so either the stop sees this process or this check
8574    // sees the roster closed: end the process now rather than start a module
8575    // the daemon is about to stop.
8576    if roster.is_closed() {
8577        // This child was never admitted, so there is no module protocol shutdown to wait for.
8578        #[cfg(target_os = "linux")]
8579        kill_module_cgroup(cgroup_placement, &cgroup_name);
8580        if let Err(error) = child.start_kill() {
8581            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8582        }
8583        #[cfg(target_os = "linux")]
8584        if let Some(placement) = cgroup_placement {
8585            // This spawn was never admitted, so shutdown has no roster entry
8586            // to await. Do not detach its cleanup: the runtime could exit
8587            // before that task reaps the rejected child and removes its group.
8588            while matches!(child.try_wait(), Ok(None)) {
8589                std::thread::yield_now();
8590            }
8591            if matches!(
8592                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8593                subc_cgroup::KillOutcome::Killed
8594            ) {
8595                if let Ok(path) = placement.module_path(&cgroup_name) {
8596                    while std::fs::read_to_string(path.join("cgroup.events"))
8597                        .ok()
8598                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8599                    {
8600                        std::thread::yield_now();
8601                    }
8602                }
8603            }
8604            remove_module_cgroup(placement, &cgroup_name);
8605        }
8606        drop(roster_guard);
8607        return Err(SuperviseError::Spawn {
8608            program: spec.program.clone(),
8609            source: io::Error::other(
8610                "the daemon began shutting down while this process was starting; ended it",
8611            ),
8612            cgroup_path,
8613        });
8614    }
8615
8616    #[cfg(windows)]
8617    if let Err(error) = resume_suspended_child(pid, spec) {
8618        kill_and_reap_rejected_child(&mut child);
8619        return Err(error);
8620    }
8621
8622    let stdout_pump = match child.stdout.take() {
8623        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8624        None => {
8625            warn!(
8626                module_id = %spec.module_id,
8627                "spawned child exposed no stdout pipe; file capture will be incomplete"
8628            );
8629            None
8630        }
8631    };
8632    let stderr_pump = match child.stderr.take() {
8633        Some(stderr) => {
8634            let generation = ring
8635                .lock()
8636                .unwrap_or_else(|poisoned| poisoned.into_inner())
8637                .begin_process();
8638            Some(StderrPump {
8639                task: tokio::spawn(pump_stderr_to(
8640                    stderr,
8641                    Arc::clone(ring),
8642                    generation,
8643                    output_sink,
8644                )),
8645                generation,
8646            })
8647        }
8648        None => {
8649            // Spawning succeeded but the pipe did not materialise. Recording it as
8650            // uncaptured keeps the tail honest: the alternative is an empty tail
8651            // that reads as a module which printed nothing.
8652            ring.lock()
8653                .unwrap_or_else(|poisoned| poisoned.into_inner())
8654                .mark_not_captured("stderr pipe was not available on spawn");
8655            warn!(
8656                module_id = %spec.module_id,
8657                "spawned child exposed no stderr pipe; tail will be unavailable"
8658            );
8659            None
8660        }
8661    };
8662
8663    let exec = Arc::new(crate::activity::ExecOutcome::new());
8664    #[cfg(target_os = "macos")]
8665    let pending_exec = privacy_exec.is_some();
8666    #[cfg(not(target_os = "macos"))]
8667    let pending_exec = false;
8668    if !pending_exec {
8669        exec.finish(crate::activity::ExecState::CommittedExec);
8670    }
8671    Ok(SupervisedChild {
8672        child,
8673        exec: crate::activity::ExecOwner(exec),
8674        #[cfg(windows)]
8675        nonce_delivery,
8676        protocol: spec.protocol,
8677        #[cfg(target_os = "linux")]
8678        module_id: cgroup_name,
8679        #[cfg(target_os = "linux")]
8680        cgroup_placement: cgroup_placement.cloned(),
8681        #[cfg(windows)]
8682        job,
8683        stdout_pump,
8684        stderr_pump,
8685        stderr_ring: Arc::clone(ring),
8686        spawned_at,
8687        spawned_at_ms,
8688        spawned_from,
8689        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8690        spawned_file_identity,
8691        process_start_time,
8692        process_identity,
8693        pid,
8694        roster_guard: Some(roster_guard),
8695        #[cfg(target_os = "macos")]
8696        privacy_exec,
8697        #[cfg(target_os = "macos")]
8698        report_ready: Arc::new(OnceLock::new()),
8699        spawn_failure: None,
8700    })
8701}
8702
8703#[cfg(test)]
8704mod spawn_record_tests {
8705    use super::*;
8706
8707    thread_local! {
8708        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8709    }
8710
8711    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8712        LAST_PID.set(child.id());
8713        #[cfg(windows)]
8714        if let Some((_, address)) = _spec
8715            .env
8716            .iter()
8717            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8718        {
8719            use std::io::{Read, Write};
8720            let mut stream = std::net::TcpStream::connect(address).unwrap();
8721            stream
8722                .write_all(&child.id().unwrap().to_ne_bytes())
8723                .unwrap();
8724            let mut release = [0];
8725            stream.read_exact(&mut release).unwrap();
8726        }
8727    }
8728
8729    #[cfg(windows)]
8730    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8731        if spec
8732            .env
8733            .iter()
8734            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8735        {
8736            Err(io::Error::other("injected module job assignment failure"))
8737        } else {
8738            Ok(())
8739        }
8740    }
8741
8742    fn spec() -> ModuleSpec {
8743        ModuleSpec {
8744            module_id: "spawn-record-worker".into(),
8745            #[cfg(unix)]
8746            program: "/bin/sleep".into(),
8747            #[cfg(windows)]
8748            program: std::env::current_exe().unwrap(),
8749            #[cfg(unix)]
8750            args: vec!["60".into()],
8751            #[cfg(windows)]
8752            args: vec![
8753                "--exact".into(),
8754                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8755                "--nocapture".into(),
8756            ],
8757            env: vec![],
8758            reserved: false,
8759            reserved_prefixes: vec![],
8760            protocol: ModuleProtocol::None,
8761            overlap: Default::default(),
8762        }
8763    }
8764
8765    #[tokio::test]
8766    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8767        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8768        let supervisor =
8769            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8770                .with_live_children_record(home.join("live-children.json"));
8771        crate::live_children::fail_next_file_sync();
8772        let result = supervisor.spawn(spec());
8773        let failed = result.is_err();
8774        if let Ok(module) = result {
8775            module.stop().await.unwrap();
8776        }
8777        assert!(failed, "record failure must fail spawn");
8778        let pid = LAST_PID.get().expect("the real OS child was created");
8779        assert!(
8780            subc_os::Process::open(pid).unwrap().is_none(),
8781            "child {pid} survived rejected admission or was not reaped"
8782        );
8783    }
8784
8785    #[cfg(target_os = "macos")]
8786    #[test]
8787    #[ignore = "manual on-device latency measurement"]
8788    fn durable_record_write_latency_50() {
8789        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8790        let path = home.join("live-children.json");
8791        let children = vec![crate::live_children::LiveChild {
8792            module_id: "latency-worker".into(),
8793            pid: std::process::id(),
8794            protocol: ModuleProtocol::None,
8795            start_time: Some(1),
8796            executable: None,
8797            cgroup_name: None,
8798        }];
8799        let mut samples = Vec::new();
8800        for _ in 0..50 {
8801            let start = std::time::Instant::now();
8802            crate::live_children::write_record(&path, &children).unwrap();
8803            samples.push(start.elapsed());
8804        }
8805        samples.sort();
8806        eprintln!(
8807            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8808            (samples[24] + samples[25]) / 2,
8809            samples[47]
8810        );
8811    }
8812
8813    #[cfg(windows)]
8814    #[test]
8815    fn windows_store_touch_fixture() {
8816        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8817            std::fs::write(path, b"module ran").unwrap();
8818        }
8819    }
8820
8821    #[cfg(windows)]
8822    #[tokio::test]
8823    async fn windows_daemon_fixture() {
8824        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8825            return;
8826        };
8827        let mut spec = spec();
8828        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8829        let supervisor =
8830            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8831        let _module = supervisor.spawn(spec).unwrap();
8832    }
8833
8834    #[cfg(windows)]
8835    #[test]
8836    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8837        use std::io::Read;
8838        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8839        // An outer job models a scheduler/service host. The daemon's root and
8840        // root job must nest without requiring breakaway from this job.
8841        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8842        command
8843            .args([
8844                "--exact",
8845                "supervise::spawn_record_tests::windows_daemon_fixture",
8846                "--nocapture",
8847            ])
8848            .env(
8849                "SUBC_TEST_DAEMON_BARRIER",
8850                listener.local_addr().unwrap().to_string(),
8851            );
8852        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8853        listener.set_nonblocking(true).unwrap();
8854        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8855        let mut stream = loop {
8856            if let Ok((stream, _)) = listener.accept() {
8857                break stream;
8858            }
8859            assert!(
8860                daemon.child.try_wait().unwrap().is_none(),
8861                "daemon failed before rendezvous"
8862            );
8863            assert!(
8864                std::time::Instant::now() < deadline,
8865                "daemon did not reach CreateProcess failpoint"
8866            );
8867            std::thread::yield_now();
8868        };
8869        stream
8870            .set_read_timeout(Some(Duration::from_secs(30)))
8871            .unwrap();
8872        let mut pid = [0; 4];
8873        stream.read_exact(&mut pid).unwrap();
8874        let pid = u32::from_ne_bytes(pid);
8875        assert!(subc_jobobject::process_exists(pid));
8876        // Terminate only the daemon, not its outer job: otherwise the host job
8877        // would also kill a child whose daemon never joined its root job.
8878        daemon.child.kill().unwrap();
8879        daemon.child.wait().unwrap();
8880        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8881        if !exited {
8882            daemon.job.terminate().unwrap();
8883        }
8884        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8885    }
8886
8887    #[cfg(windows)]
8888    #[tokio::test]
8889    async fn windows_containment_failure_never_resumes() {
8890        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8891        let sentinel = home.join("store-touched");
8892        let mut spec = spec();
8893        spec.env
8894            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8895        spec.env.push((
8896            "SUBC_TEST_STORE_TOUCH".into(),
8897            sentinel.display().to_string(),
8898        ));
8899        let supervisor =
8900            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8901        let result = supervisor.spawn(spec);
8902        // A version that resumes despite assignment failure must have time to
8903        // execute its sentinel before dropping the returned module kills it.
8904        if let Ok(module) = result {
8905            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8906                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8907                    tokio::task::yield_now().await;
8908                }
8909            })
8910            .await;
8911            let _ = module.stop().await;
8912            panic!(
8913                "assignment failure resumed a module; store touched={}",
8914                sentinel.exists()
8915            );
8916        }
8917        assert!(
8918            !sentinel.exists(),
8919            "refused module executed store-touch sentinel"
8920        );
8921        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8922    }
8923
8924    #[cfg(windows)]
8925    #[tokio::test]
8926    async fn windows_nested_daemon_fixture() {
8927        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8928            return;
8929        };
8930        let mut spec = spec();
8931        spec.env.push((
8932            "SUBC_TEST_STORE_TOUCH".into(),
8933            sentinel.to_str().unwrap().into(),
8934        ));
8935        let supervisor =
8936            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8937        let module = supervisor
8938            .spawn(spec)
8939            .expect("per-module job must nest below daemon and host jobs");
8940        tokio::time::timeout(Duration::from_secs(10), async {
8941            while !PathBuf::from(&sentinel).exists() {
8942                tokio::task::yield_now().await;
8943            }
8944        })
8945        .await
8946        .unwrap();
8947        module.stop().await.unwrap();
8948    }
8949
8950    #[cfg(windows)]
8951    #[test]
8952    fn windows_module_job_nests_under_host_job() {
8953        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8954        let sentinel = home.join("nested-store-touched");
8955        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8956        command
8957            .args([
8958                "--exact",
8959                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8960                "--nocapture",
8961            ])
8962            .env("SUBC_TEST_NESTED_STORE", &sentinel);
8963        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8964        assert!(daemon.child.wait().unwrap().success());
8965        assert!(sentinel.exists(), "nested protocol:none module never ran");
8966    }
8967}
8968
8969#[cfg(target_os = "linux")]
8970pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8971    use subc_cgroup::KillOutcome;
8972    match subc_cgroup::kill_module(placement, module_id) {
8973        KillOutcome::Killed => {}
8974        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8975            debug!(
8976                module_id,
8977                "cgroup tree kill unavailable; using direct-child kill"
8978            );
8979        }
8980        KillOutcome::IoError { path, error } => {
8981            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8982        }
8983    }
8984}
8985
8986/// Assign a freshly spawned Windows child without ever resuming on failure.
8987///
8988/// Steps 2 and 3 of the suspended-create contract: the job is created and the
8989/// child assigned **while it is still suspended** (step 1 is
8990/// `suspend_on_create_async` at the spawn site). The caller resumes only after
8991/// the child's roster admission is durable. Failed containment kills and reaps
8992/// the suspended child instead of running an unprotected module.
8993#[cfg(windows)]
8994fn contain_spawned_child(
8995    child: &mut Child,
8996    spec: &ModuleSpec,
8997) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8998    let assignment = || {
8999        let job = subc_jobobject::JobObject::new()?;
9000        #[cfg(test)]
9001        spawn_record_tests::assignment_failpoint(spec)?;
9002        job.assign(child)?;
9003        Ok::<_, io::Error>(job)
9004    };
9005    match assignment() {
9006        Ok(job) => Ok(Some(job)),
9007        Err(source) => {
9008            kill_and_reap_rejected_child(child);
9009            Err(SuperviseError::Spawn {
9010                program: spec.program.clone(),
9011                source: io::Error::other(format!(
9012                    "Windows module job containment failed: {source}"
9013                )),
9014                cgroup_path: None,
9015            })
9016        }
9017    }
9018}
9019
9020/// Rejection is synchronous: do not return a spawn error while the child can
9021/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9022fn kill_and_reap_rejected_child(child: &mut Child) {
9023    #[cfg(unix)]
9024    if let Some(pid) = child
9025        .id()
9026        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9027    {
9028        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9029    }
9030    let _ = child.start_kill();
9031    while matches!(child.try_wait(), Ok(None)) {
9032        std::thread::yield_now();
9033    }
9034}
9035
9036/// Resume a suspended child, killing it if it cannot be started.
9037///
9038/// A suspended process holds a pid and does nothing, so there is no useful
9039/// state to return: the caller gets an error and the spawn fails.
9040#[cfg(windows)]
9041fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9042    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9043        // Kill it here rather than leaving a suspended process for the caller
9044        // to notice; `kill_on_drop` would eventually do this, but the module
9045        // would have been reported as running in between.
9046        let _ = std::process::Command::new("taskkill.exe")
9047            .args(["/PID", &pid.to_string(), "/T", "/F"])
9048            .stdin(Stdio::null())
9049            .stdout(Stdio::null())
9050            .stderr(Stdio::null())
9051            .status();
9052        return Err(SuperviseError::Spawn {
9053            program: spec.program.clone(),
9054            source,
9055            cgroup_path: None,
9056        });
9057    }
9058    Ok(())
9059}
9060
9061#[cfg(target_os = "linux")]
9062fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9063    match placement.remove_module(module_id) {
9064        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9065        Err(error) => warn!(
9066            module_id,
9067            error = %error,
9068            "could not remove module cgroup after process exit; continuing teardown"
9069        ),
9070    }
9071}
9072
9073#[cfg(target_os = "linux")]
9074async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9075    // Reaping the direct child is not proof its descendants exited. End the
9076    // residual tree and wait for the kernel's population fact before rmdir;
9077    // otherwise a successful parent wait leaks a directory on each restart.
9078    if matches!(
9079        subc_cgroup::kill_module(Some(placement), module_id),
9080        subc_cgroup::KillOutcome::Killed
9081    ) {
9082        if let Ok(path) = placement.module_path(module_id) {
9083            while std::fs::read_to_string(path.join("cgroup.events"))
9084                .ok()
9085                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9086            {
9087                sleep(Duration::from_millis(1)).await;
9088            }
9089        }
9090    }
9091    remove_module_cgroup(placement, module_id);
9092}
9093
9094#[cfg(target_os = "linux")]
9095fn apply_cgroup_placement(
9096    command: &mut Command,
9097    spec: &ModuleSpec,
9098    path: &std::path::Path,
9099) -> Result<(), SuperviseError> {
9100    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9101        module_id: spec.module_id.clone(),
9102        source,
9103    })
9104}
9105
9106fn capture_retention(spec: &ModuleSpec) -> Retention {
9107    let defaults = Retention::default();
9108    let value = |name: &str| {
9109        spec.env
9110            .iter()
9111            .rev()
9112            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9113    };
9114    Retention {
9115        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9116            .and_then(|value| value.parse().ok())
9117            .unwrap_or(defaults.max_file_mb),
9118        keep: value(CAPTURE_KEEP_ENV)
9119            .and_then(|value| value.parse().ok())
9120            .unwrap_or(defaults.keep),
9121        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9122            .and_then(|value| value.parse().ok())
9123            .unwrap_or(defaults.max_age_days),
9124    }
9125}
9126
9127/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9128/// module's registration to the exact process the supervisor spawned.
9129fn generate_launch_nonce() -> Result<String, SuperviseError> {
9130    let mut bytes = [0u8; 32];
9131    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9132        reason: source.to_string(),
9133    })?;
9134    let mut hex = String::with_capacity(64);
9135    for b in bytes {
9136        use std::fmt::Write;
9137        let _ = write!(hex, "{b:02x}");
9138    }
9139    Ok(hex)
9140}
9141
9142/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9143/// signal about how many leading bytes matched.
9144fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9145    if a.len() != b.len() {
9146        return false;
9147    }
9148    let mut diff = 0u8;
9149    for (x, y) in a.iter().zip(b.iter()) {
9150        diff |= x ^ y;
9151    }
9152    diff == 0
9153}
9154
9155/// The kernel's image after an acknowledged exec, shared by ordinary launches
9156/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9157/// not identities inferred from a configured pathname.
9158#[cfg(not(windows))]
9159fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9160    let process = subc_os::Process::open(pid).ok().flatten()?;
9161    #[cfg(target_os = "linux")]
9162    return sample_after_linux_exec(
9163        || {
9164            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9165            let committed = linux_exec_committed(&stat);
9166            #[cfg(test)]
9167            spawn_image_tests::report_exec_state(committed);
9168            committed
9169        },
9170        || process.observe(),
9171    );
9172    #[cfg(not(target_os = "linux"))]
9173    process.observe()
9174}
9175
9176/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9177/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9178/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9179#[cfg(target_os = "linux")]
9180fn linux_exec_committed(stat: &str) -> Option<bool> {
9181    const PF_FORKNOEXEC: u64 = 0x40;
9182    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9183    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9184}
9185
9186#[cfg(target_os = "linux")]
9187fn sample_after_linux_exec(
9188    mut exec_committed: impl FnMut() -> Option<bool>,
9189    observe: impl FnOnce() -> Option<subc_os::Observation>,
9190) -> Option<subc_os::Observation> {
9191    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9192    while !exec_committed()? {
9193        // A failed or indefinitely stalled exec must not publish the inherited
9194        // image as authority to signal a future orphan. Missing identity keeps
9195        // the existing conservative orphan-sweep behavior.
9196        if std::time::Instant::now() >= deadline {
9197            return None;
9198        }
9199        std::thread::sleep(Duration::from_micros(50));
9200    }
9201    observe()
9202}
9203
9204#[cfg(all(test, target_os = "linux"))]
9205mod spawn_image_tests {
9206    use super::*;
9207    use std::cell::{Cell, RefCell};
9208
9209    enum ExecEvent {
9210        Uncommitted,
9211        Image(Option<subc_os::Observation>),
9212    }
9213
9214    thread_local! {
9215        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9216    }
9217
9218    pub(super) fn report_exec_state(committed: Option<bool>) {
9219        if committed == Some(false) {
9220            EXEC_EVENTS.with(|events| {
9221                if let Some(events) = events.borrow_mut().take() {
9222                    let _ = events.send(ExecEvent::Uncommitted);
9223                }
9224            });
9225        }
9226    }
9227
9228    #[test]
9229    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9230        use std::io::{Read, Write};
9231        use std::os::fd::AsRawFd;
9232
9233        let descriptor = std::fs::File::open("/dev/null").unwrap();
9234        let mut command = std::process::Command::new("/bin/sleep");
9235        command.arg("30");
9236        let (mut ready, mut release) =
9237            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9238                .unwrap();
9239        let spawn = std::thread::spawn(move || command.spawn());
9240        let mut message = [0u8; 8];
9241        ready.read_exact(&mut message).unwrap();
9242        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9243        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9244        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9245        let (events, received) = std::sync::mpsc::channel();
9246        let observer = std::thread::spawn(move || {
9247            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9248            let image = observe_spawned_image(pid);
9249            let _ = events.send(ExecEvent::Image(image));
9250        });
9251        // Release only once the production observer has seen the uncommitted
9252        // exec, or has incorrectly returned the inherited image. Either way,
9253        // release and reap before asserting so a regression cannot leak a child.
9254        let first = received.recv_timeout(Duration::from_secs(5));
9255        release.write_all(&[1]).unwrap();
9256        let mut child = spawn.join().unwrap().unwrap();
9257        let after = match first {
9258            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9259                Ok(ExecEvent::Image(image)) => image,
9260                _ => None,
9261            },
9262            Ok(ExecEvent::Image(image)) => image,
9263            Err(_) => None,
9264        };
9265        observer.join().unwrap();
9266        child.kill().unwrap();
9267        child.wait().unwrap();
9268
9269        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9270        assert_eq!(
9271            inherited.unwrap().executable,
9272            subc_os::file_identity(&std::env::current_exe().unwrap())
9273        );
9274        assert_eq!(
9275            after.unwrap().executable,
9276            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9277        );
9278    }
9279
9280    #[test]
9281    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9282        let committed = Cell::new(false);
9283        let inherited = subc_os::Observation {
9284            start_time: 23,
9285            executable: Some(subc_os::FileIdentity {
9286                device: 29,
9287                inode: 31,
9288            }),
9289        };
9290        let module = subc_os::Observation {
9291            start_time: 23,
9292            executable: Some(subc_os::FileIdentity {
9293                device: 29,
9294                inode: 37,
9295            }),
9296        };
9297        let polls = Cell::new(0);
9298        let image = sample_after_linux_exec(
9299            || {
9300                polls.set(polls.get() + 1);
9301                // Hold the inherited image through the first poll, as in the
9302                // posix_spawn parent-wake window, then commit the module image.
9303                committed.set(polls.get() == 2);
9304                Some(committed.get())
9305            },
9306            || Some(if committed.get() { module } else { inherited }),
9307        );
9308        assert_eq!(image, Some(module));
9309        assert_eq!(polls.get(), 2);
9310    }
9311
9312    #[test]
9313    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9314        assert_eq!(
9315            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9316            Some(false)
9317        );
9318        assert_eq!(
9319            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9320            Some(true)
9321        );
9322        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9323        assert_eq!(linux_exec_committed("17 (a) R"), None);
9324    }
9325
9326    #[test]
9327    fn unreadable_exec_state_never_samples_an_inherited_image() {
9328        assert_eq!(
9329            sample_after_linux_exec(
9330                || None,
9331                || panic!("must not sample before exec confirmation")
9332            ),
9333            None
9334        );
9335    }
9336
9337    #[test]
9338    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9339        assert_eq!(
9340            sample_after_linux_exec(
9341                || Some(false),
9342                || panic!("must not sample after an exec timeout")
9343            ),
9344            None
9345        );
9346    }
9347}
9348
9349/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9350/// different executable after the no-delete capture was taken.
9351#[cfg(windows)]
9352fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9353    let mut program = spec.program.clone();
9354    if program.is_absolute() || program.components().count() > 1 {
9355        // An explicit path can name a PE image with no extension. Try that
9356        // exact file first, as Command does, before falling back to .exe.
9357        match subc_os::ExecutableCapture::open(&program) {
9358            Ok(capture) => return Ok(capture),
9359            Err(error)
9360                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9361            Err(error) => return Err(error),
9362        }
9363        program.set_extension("exe");
9364        return subc_os::ExecutableCapture::open(&program);
9365    }
9366    if program.extension().is_none() {
9367        program.set_extension("exe");
9368    }
9369    let custom_path = spec
9370        .env
9371        .iter()
9372        .rev()
9373        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9374    let mut directories = Vec::new();
9375    // These are Windows' default executable search locations. A configured
9376    // PATH still overrides the inherited PATH, not the system directories.
9377    if let Some(parent) = std::env::current_exe()?.parent() {
9378        directories.push(parent.to_path_buf());
9379    }
9380    directories.push(std::env::current_dir()?);
9381    if let Some(windows) = std::env::var_os("SystemRoot") {
9382        let windows = PathBuf::from(windows);
9383        directories.push(windows.join("System32"));
9384        directories.push(windows.clone());
9385    }
9386    let path = custom_path
9387        .map(|(_, value)| std::ffi::OsString::from(value))
9388        .or_else(|| std::env::var_os("PATH"));
9389    if let Some(path) = path {
9390        directories.extend(std::env::split_paths(&path));
9391    }
9392    for directory in directories {
9393        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9394            Ok(capture) => return Ok(capture),
9395            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9396            Err(error) => return Err(error),
9397        }
9398    }
9399    Err(io::Error::new(
9400        io::ErrorKind::NotFound,
9401        "resolved executable not found",
9402    ))
9403}
9404
9405fn spawn_and_mark_running(
9406    spec: &ModuleSpec,
9407    runtime: &SupervisorRuntimeConfig,
9408    snapshot: &SharedSnapshot,
9409) -> Result<SupervisedChild, SuperviseError> {
9410    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9411    let child = spawn_child(
9412        spec,
9413        runtime.connection_file_path.as_deref(),
9414        runtime.supervisor_handle.as_ref(),
9415        &runtime.stderr_ring,
9416        runtime.capture_logs_dir.as_deref(),
9417        &runtime.child_roster,
9418        #[cfg(target_os = "linux")]
9419        runtime.cgroup_placement.as_ref(),
9420    )?;
9421    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9422    if let Some(spawn) = activity_spawn {
9423        spawn.complete(snapshot);
9424    }
9425    Ok(child)
9426}
9427
9428enum RegistrationWaitOutcome {
9429    Registered,
9430    Exited(ExitReport),
9431    TimedOut,
9432}
9433
9434struct ReloadRegistrationFailure {
9435    exit_report: ExitReport,
9436    reason: String,
9437}
9438
9439#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9440enum BusyGaugeObservation {
9441    Quiescent,
9442    Busy,
9443    Omitted,
9444}
9445
9446/// What one health answer said about one declared busy gauge.
9447#[derive(Debug, Clone, PartialEq, Eq)]
9448enum GaugeVerdict {
9449    /// The gauge read 0: this gauge is not what holds the drain.
9450    Zero,
9451    /// The gauge read a positive count.
9452    NonZero(u64),
9453    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9454    /// a negative number). It counts as busy, because the daemon cannot tell
9455    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9456    NotACount(String),
9457    /// The health answer had no metrics object, or one without this gauge.
9458    Missing,
9459    /// The probe produced no health answer to read; carries the probe error's
9460    /// evidence label (`no-answer` for a probe that timed out).
9461    ProbeFailed(&'static str),
9462}
9463
9464impl GaugeVerdict {
9465    fn reads_quiet(&self) -> bool {
9466        matches!(self, Self::Zero)
9467    }
9468}
9469
9470impl fmt::Display for GaugeVerdict {
9471    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9472        match self {
9473            Self::Zero => f.write_str("zero"),
9474            Self::NonZero(value) => write!(f, "non_zero({value})"),
9475            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9476            Self::Missing => f.write_str("missing_from_health_body"),
9477            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9478        }
9479    }
9480}
9481
9482/// The longest JSON text kept for a gauge value that is not a count, so a
9483/// module reporting a large object as a gauge cannot flood the drain log.
9484const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9485
9486/// Read each declared gauge out of one health answer's metrics.
9487fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9488    let metrics = metrics.and_then(Value::as_object);
9489    gauges
9490        .iter()
9491        .map(
9492            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9493                None => GaugeVerdict::Missing,
9494                Some(value) => match value.as_u64() {
9495                    Some(0) => GaugeVerdict::Zero,
9496                    Some(count) => GaugeVerdict::NonZero(count),
9497                    None => {
9498                        let mut text = value.to_string();
9499                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9500                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9501                            while !text.is_char_boundary(end) {
9502                                end -= 1;
9503                            }
9504                            text.truncate(end);
9505                            text.push_str("...");
9506                        }
9507                        GaugeVerdict::NotACount(text)
9508                    }
9509                },
9510            },
9511        )
9512        .collect()
9513}
9514
9515/// The drain's overall reading of one health answer: omitted when any
9516/// declared gauge is missing, quiescent only when every gauge reads 0.
9517fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9518    if verdicts
9519        .iter()
9520        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9521    {
9522        BusyGaugeObservation::Omitted
9523    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9524        BusyGaugeObservation::Quiescent
9525    } else {
9526        BusyGaugeObservation::Busy
9527    }
9528}
9529
9530/// One declared gauge's history over a drain: the latest verdict on it, and
9531/// the latest value actually read for it (which a later failed probe or
9532/// missing gauge does not erase). Times are offsets from the start of the
9533/// drain's wait.
9534#[derive(Debug, Clone, Default, PartialEq, Eq)]
9535struct GaugeTrace {
9536    latest: Option<(GaugeVerdict, Duration)>,
9537    last_value: Option<(GaugeVerdict, Duration)>,
9538}
9539
9540/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9541/// its ceiling can say in its timeout line which declared gauge held it and
9542/// why. Without it, a gauge that read busy, a gauge the module never
9543/// reported, and a probe that never got an answer all looked the same: a
9544/// drain that ended at its ceiling with nothing in flight.
9545#[derive(Debug, Clone, Default, PartialEq, Eq)]
9546struct DrainGaugeTrace {
9547    /// Health probes the drain sent.
9548    probes: u32,
9549    /// Each declared gauge, in declaration order.
9550    gauges: Vec<(String, GaugeTrace)>,
9551}
9552
9553impl DrainGaugeTrace {
9554    fn new(gauges: &[String]) -> Self {
9555        Self {
9556            probes: 0,
9557            gauges: gauges
9558                .iter()
9559                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9560                .collect(),
9561        }
9562    }
9563
9564    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9565    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9566        self.probes = self.probes.saturating_add(1);
9567        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9568            trace.latest = Some((verdict.clone(), at));
9569            if matches!(
9570                verdict,
9571                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9572            ) {
9573                trace.last_value = Some((verdict.clone(), at));
9574            }
9575        }
9576    }
9577}
9578
9579/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
9580/// declared gauge, separated by ` | `. A value is shown as its verdict
9581/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
9582/// and whether that counted as busy.
9583impl fmt::Display for DrainGaugeTrace {
9584    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9585        if self.gauges.is_empty() {
9586            return f.write_str("none declared");
9587        }
9588        for (index, (name, trace)) in self.gauges.iter().enumerate() {
9589            if index > 0 {
9590                f.write_str(" | ")?;
9591            }
9592            write!(f, "{name}: ")?;
9593            match &trace.latest {
9594                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9595                None => f.write_str("never probed")?,
9596            }
9597            match &trace.last_value {
9598                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9599                None => f.write_str("; no value ever read")?,
9600            }
9601        }
9602        Ok(())
9603    }
9604}
9605
9606fn declared_busy_gauges(
9607    registry: &Registry,
9608    module_id: &str,
9609) -> Result<Vec<String>, SuperviseError> {
9610    busy_gauges_of(
9611        registry
9612            .get_module(module_id)
9613            .map_err(SuperviseError::Registry)?,
9614    )
9615}
9616
9617/// [`declared_busy_gauges`] for the registration a connection holds, in any
9618/// slot: after cutover the incumbent is no longer the id's active
9619/// registration, and its own manifest is the one that names its gauges.
9620fn declared_busy_gauges_for_connection(
9621    registry: &Registry,
9622    connection_id: ConnectionId,
9623) -> Result<Vec<String>, SuperviseError> {
9624    busy_gauges_of(
9625        registry
9626            .get_module_by_connection(connection_id)
9627            .map_err(SuperviseError::Registry)?,
9628    )
9629}
9630
9631fn busy_gauges_of(
9632    registration: Option<crate::registry::ModuleRegistration>,
9633) -> Result<Vec<String>, SuperviseError> {
9634    let Some(registration) = registration else {
9635        return Ok(Vec::new());
9636    };
9637    let Some(self_signals) = registration.manifest.self_signals else {
9638        return Ok(Vec::new());
9639    };
9640
9641    let mut gauges = Vec::new();
9642    for declaration in self_signals {
9643        if declaration.kind != SelfSignalKind::Busy {
9644            continue;
9645        }
9646        match declaration.anchored_to {
9647            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9648                gauges.extend(declared)
9649            }
9650            _ => {
9651                // An invalid Busy anchor is fail-safe: the empty name cannot be
9652                // present in a conforming health report, so this drain stays busy.
9653                gauges.push(String::new());
9654            }
9655        }
9656    }
9657    Ok(gauges)
9658}
9659
9660/// Wait for `endpoint` to have nothing in flight and, when the module declares
9661/// busy gauges, for a health probe to report them quiet. The probe is addressed
9662/// by `scope`: a swap's superseded incumbent must be asked about its own
9663/// gauges, and by module id the probe would reach the promoted candidate.
9664/// While the gauges read busy, the probe repeats every
9665/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9666/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
9667/// timeout line.
9668#[allow(clippy::too_many_arguments)]
9669async fn wait_for_forwarding_quiescence(
9670    forwarding: &ForwardingTable,
9671    module_id: &str,
9672    runtime: &SupervisorRuntimeConfig,
9673    endpoint: crate::ModuleEndpointId,
9674    deadline: Instant,
9675    busy_gauges: &[String],
9676    scope: DrainScope,
9677    trace: &mut DrainGaugeTrace,
9678) -> Result<bool, SuperviseError> {
9679    let wait_started = Instant::now();
9680    let mut gauges_quiescent = busy_gauges.is_empty();
9681    let mut next_probe_at = wait_started;
9682    let mut omission_reported = false;
9683
9684    loop {
9685        let now = Instant::now();
9686        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9687            let report = match scope {
9688                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9689                DrainScope::Endpoint(endpoint) => {
9690                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9691                }
9692            };
9693            let verdicts = match &report {
9694                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9695                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9696            };
9697            trace.record(&verdicts, wait_started.elapsed());
9698            gauges_quiescent = match report {
9699                Ok(_) => match busy_gauge_observation(&verdicts) {
9700                    BusyGaugeObservation::Quiescent => true,
9701                    BusyGaugeObservation::Busy => false,
9702                    BusyGaugeObservation::Omitted => {
9703                        // Once per drain: a module that never reports a gauge
9704                        // it declared would otherwise say so on every probe.
9705                        if !omission_reported {
9706                            forwarding
9707                                .counters()
9708                                .increment_drains_with_undeclared_gauge();
9709                            let missing = busy_gauges
9710                                .iter()
9711                                .zip(&verdicts)
9712                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9713                                .map(|(gauge, _)| gauge.as_str())
9714                                .collect::<Vec<_>>()
9715                                .join(",");
9716                            warn!(
9717                                module_id,
9718                                missing_gauges = %missing,
9719                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9720                            );
9721                            omission_reported = true;
9722                        }
9723                        false
9724                    }
9725                },
9726                Err(err) => {
9727                    warn!(
9728                        module_id,
9729                        error = %err,
9730                        "drain health.check did not produce declared busy gauges; treating module as busy"
9731                    );
9732                    false
9733                }
9734            };
9735            // Counted from the probe's completion: the probe above was awaited,
9736            // so no second probe can start while one is still outstanding.
9737            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9738        }
9739
9740        let in_flight = forwarding
9741            .endpoint_in_flight_count(endpoint)
9742            .map_err(SuperviseError::Forwarding)?;
9743        if in_flight == 0 && gauges_quiescent {
9744            return Ok(true);
9745        }
9746
9747        let now = Instant::now();
9748        if now >= deadline {
9749            return Ok(false);
9750        }
9751        let mut wait = deadline
9752            .saturating_duration_since(now)
9753            .min(REGISTRY_RELEASE_POLL);
9754        if !busy_gauges.is_empty() {
9755            wait = wait.min(next_probe_at.saturating_duration_since(now));
9756        }
9757        sleep(wait).await;
9758    }
9759}
9760
9761/// The wait between drain busy-gauge probes: the larger of
9762/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
9763/// drain's own in-flight poll interval.
9764fn drain_busy_reprobe_interval() -> Duration {
9765    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9766}
9767
9768/// Logs why a drain gave up: the drain waited its full timeout without the
9769/// module going quiet, and teardown is being forced.
9770///
9771/// The line names two kinds of evidence, because a bare "timed out" hides
9772/// causes that need different fixes:
9773/// - `holdouts`: the requests and routes still open when the timeout hit,
9774///   and on which connections. One stuck call is a module bug; a held stream
9775///   on every session is a module that should end its streams when it
9776///   receives `module.draining`.
9777/// - `gauges`: for each busy gauge the module declares, its latest reading
9778///   and how many probes ran. This separates a module that kept reporting
9779///   itself busy, one whose health answer never contained the gauge, and one
9780///   that never answered the probes, which otherwise all look like a timeout
9781///   with nothing in flight.
9782fn warn_drain_timed_out(
9783    module_id: &str,
9784    drain_timeout: Duration,
9785    reason: RouteCloseReason,
9786    holdouts: &crate::forwarding::DrainHoldouts,
9787    gauges: &DrainGaugeTrace,
9788) {
9789    warn!(
9790        module_id = %module_id,
9791        waited = ?drain_timeout,
9792        ?reason,
9793        held_requests = holdouts.requests,
9794        held_routes = holdouts.routes,
9795        total_routes = holdouts.total_routes,
9796        top_connections = ?holdouts.top_connections,
9797        // `module_channel:corr`, so the module can find each held request
9798        // in its own log; capped, so `held_requests` is the full count.
9799        held = %holdouts
9800            .held
9801            .iter()
9802            .map(|(channel, corr)| format!("{channel}:{corr}"))
9803            .collect::<Vec<_>>()
9804            .join(","),
9805        gauge_probes = gauges.probes,
9806        busy_gauges = %gauges,
9807        "route drain timed out before request quiescence; forcing teardown"
9808    );
9809}
9810
9811/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9812///
9813/// `Ok` is always honest and passed straight through -- the wait actually measured
9814/// in-flight state. `Err` means the wait produced no measurement at all (the
9815/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9816/// constant: the drain did not complete. Never recomputed from route state, never a
9817/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9818fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9819    match wait_result {
9820        Ok(drained) => *drained,
9821        Err(_) => false,
9822    }
9823}
9824
9825fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9826    for released in released_routes {
9827        let frame = match Frame::build_with_version(
9828            released.negotiated_ver,
9829            FrameType::Goodbye,
9830            control_flags(),
9831            released.channel,
9832            released.epoch,
9833            0,
9834            Vec::new(),
9835        ) {
9836            Ok(frame) => frame,
9837            Err(err) => {
9838                warn!(
9839                    route_channel = released.channel,
9840                    error = %err,
9841                    "failed to build supervisor drain route GOODBYE frame"
9842                );
9843                continue;
9844            }
9845        };
9846        if !released.close_on_delivery_failure() {
9847            crate::forwarding::send_module_route_goodbye(
9848                &forwarding.counters(),
9849                &released.sink,
9850                frame,
9851                released.module_id.as_deref(),
9852                "supervisor drain",
9853            );
9854            continue;
9855        }
9856        if let Err(err) = released.sink.try_send(frame) {
9857            warn!(
9858                target_connection_id = released.connection_id.get(),
9859                route_channel = released.channel,
9860                error = %err,
9861                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9862            );
9863            let _ = forwarding.escalate_client_delivery_failure(
9864                released.connection_id,
9865                released.channel,
9866                released.epoch,
9867                CloseReason::new(
9868                    "route_goodbye_delivery_failed",
9869                    format!(
9870                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9871                        released.channel
9872                    ),
9873                ),
9874                crate::forwarding::UndeliveredFrame {
9875                    module_id: released.module_id.as_deref(),
9876                    sink: &released.sink,
9877                },
9878            );
9879        }
9880    }
9881}
9882
9883fn send_module_draining(
9884    module_id: &str,
9885    reason: RouteCloseReason,
9886    deadline_ms: u64,
9887    target: &ModuleDrainTarget,
9888) {
9889    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9890        reason,
9891        deadline_ms,
9892    }) {
9893        Ok(body) => body,
9894        Err(err) => {
9895            warn!(
9896                module_id,
9897                error = %err,
9898                "failed to encode module draining command"
9899            );
9900            return;
9901        }
9902    };
9903    let frame = match Frame::build_with_version(
9904        target.negotiated_ver,
9905        FrameType::Push,
9906        control_flags(),
9907        0,
9908        0,
9909        0,
9910        body,
9911    ) {
9912        Ok(frame) => frame,
9913        Err(err) => {
9914            warn!(
9915                module_id,
9916                error = %err,
9917                "failed to build module draining command frame"
9918            );
9919            return;
9920        }
9921    };
9922    if let Err(err) = target.sink.try_send(frame) {
9923        warn!(
9924            module_id,
9925            target_connection_id = target.endpoint.connection_id.get(),
9926            error = %err,
9927            "module draining command was not delivered to peer"
9928        );
9929    }
9930}
9931
9932/// The channel-0 GOODBYE that tells a module its stop is planned.
9933fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9934    match Frame::build_with_version(
9935        negotiated_ver,
9936        FrameType::Goodbye,
9937        control_flags(),
9938        0,
9939        0,
9940        0,
9941        Vec::new(),
9942    ) {
9943        Ok(frame) => Some(frame),
9944        Err(err) => {
9945            warn!(
9946                module_id,
9947                error = %err,
9948                "failed to build module GOODBYE frame"
9949            );
9950            None
9951        }
9952    }
9953}
9954
9955/// Send every registered module connection its module GOODBYE at daemon
9956/// shutdown, then request that connection's close.
9957///
9958/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
9959/// before EOF, so the GOODBYE must reach the socket before the close. A close
9960/// request does not wait for the connection's queued frames: its writer gets a
9961/// bounded grace after the close, is aborted if it overruns it, and the daemon
9962/// process may exit before that grace ends. So with `wait_for_flush`, each
9963/// connection is closed only after its writer has acknowledged writing the
9964/// GOODBYE, or once a short shared budget runs out, so one module that is not
9965/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
9966/// are only queued, for a shutdown the operator has told to stop waiting.
9967/// A connection that is already gone is skipped.
9968#[cfg(unix)]
9969async fn send_module_goodbyes_for_daemon_shutdown(
9970    forwarding: &Arc<ForwardingTable>,
9971    reason: &CloseReason,
9972    wait_for_flush: bool,
9973) {
9974    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9975    let targets = match forwarding.module_connections() {
9976        Ok(targets) => targets,
9977        Err(err) => {
9978            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9979            return;
9980        }
9981    };
9982    let deadline = Instant::now() + GOODBYE_BUDGET;
9983    let mut sends = tokio::task::JoinSet::new();
9984    for target in targets {
9985        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9986            continue;
9987        };
9988        if !wait_for_flush {
9989            if let Err(err) = target.sink.try_send(frame) {
9990                debug!(
9991                    module_id = %target.module_id,
9992                    error = %err,
9993                    "shutdown module GOODBYE was not queued"
9994                );
9995            }
9996            continue;
9997        }
9998        let forwarding = Arc::clone(forwarding);
9999        let reason = reason.clone();
10000        sends.spawn(async move {
10001            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10002                Ok(Ok(())) => {}
10003                Ok(Err(err)) => debug!(
10004                    module_id = %target.module_id,
10005                    error = %err,
10006                    "module connection closed before its shutdown GOODBYE was written"
10007                ),
10008                Err(_) => warn!(
10009                    module_id = %target.module_id,
10010                    budget = ?GOODBYE_BUDGET,
10011                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10012                ),
10013            }
10014            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10015        });
10016    }
10017    // Every task ends by the shared deadline, so this wait is bounded too.
10018    while sends.join_next().await.is_some() {}
10019}
10020
10021fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10022    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10023        return;
10024    };
10025    if let Err(err) = target.sink.try_send(frame) {
10026        warn!(
10027            module_id,
10028            target_connection_id = target.endpoint.connection_id.get(),
10029            error = %err,
10030            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10031        );
10032        forwarding.request_connection_close(
10033            target.endpoint.connection_id,
10034            CloseReason::new(
10035                "module_goodbye_delivery_failed",
10036                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10037            ),
10038        );
10039    }
10040}
10041
10042#[derive(Clone, Copy)]
10043struct ForwardingDrainContext<'a> {
10044    spec: &'a ModuleSpec,
10045    runtime: &'a SupervisorRuntimeConfig,
10046    registry: &'a Registry,
10047    scope: DrainScope,
10048}
10049
10050/// Which process a forwarding drain addresses.
10051#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10052enum DrainScope {
10053    /// Whatever endpoint is active for the module id: every plain stop,
10054    /// restart and reload. Also moves the module's state to `Draining`.
10055    Active,
10056    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10057    /// module id would resolve to the promoted candidate and leave neither
10058    /// process routable. The module's state is left alone, since the promoted
10059    /// candidate is what it describes and that process is running.
10060    Endpoint(crate::ModuleEndpointId),
10061}
10062
10063/// Whether a child being drained has already been asked to stop by the time
10064/// its drain wait starts.
10065///
10066/// The drain wait is the same budget whatever this says. What it decides is
10067/// whether the supervisor must ask by signal before that wait begins: a child
10068/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10069/// healthy or not.
10070#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10071enum StopNotice {
10072    /// The module was sent `module.draining` and a module GOODBYE over its own
10073    /// registered connection, and stops itself.
10074    SentOverConnection,
10075    /// The forwarding drain found no registered connection for the module: a
10076    /// subc child spawned moments ago that has not sent HELLO yet, or a
10077    /// `protocol: "none"` child, which never registers.
10078    NoConnection,
10079    /// This path sends nothing over the module's connection: the supervisor has
10080    /// no forwarding table, or the caller stops the child without a forwarding
10081    /// drain.
10082    NotSent,
10083}
10084
10085async fn begin_forwarding_drain(
10086    spec: &ModuleSpec,
10087    runtime: &SupervisorRuntimeConfig,
10088    registry: &Registry,
10089    snapshot: &SharedSnapshot,
10090    enabled: Option<bool>,
10091    reason: RouteCloseReason,
10092) -> Result<StopNotice, SuperviseError> {
10093    let Some(forwarding) = runtime.forwarding.as_ref() else {
10094        return Err(SuperviseError::ReloadUnavailable {
10095            module_id: spec.module_id.clone(),
10096            reason: "supervisor was not configured with a forwarding table".to_string(),
10097        });
10098    };
10099
10100    begin_forwarding_drain_with(
10101        forwarding,
10102        ForwardingDrainContext {
10103            spec,
10104            runtime,
10105            registry,
10106            scope: DrainScope::Active,
10107        },
10108        snapshot,
10109        enabled,
10110        reason,
10111        runtime.drain_timeout,
10112    )
10113    .await
10114}
10115
10116async fn begin_forwarding_drain_if_configured(
10117    spec: &ModuleSpec,
10118    runtime: &SupervisorRuntimeConfig,
10119    registry: &Registry,
10120    snapshot: &SharedSnapshot,
10121    enabled: Option<bool>,
10122    reason: RouteCloseReason,
10123) -> Result<StopNotice, SuperviseError> {
10124    begin_forwarding_drain_with_timeout(
10125        spec,
10126        runtime,
10127        registry,
10128        snapshot,
10129        enabled,
10130        reason,
10131        runtime.drain_timeout,
10132    )
10133    .await
10134}
10135
10136/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10137/// budget, for paths where the operator overrides the module's configured one
10138/// (`supervisor.restart{drain_timeout_ms}`).
10139async fn begin_forwarding_drain_with_timeout(
10140    spec: &ModuleSpec,
10141    runtime: &SupervisorRuntimeConfig,
10142    registry: &Registry,
10143    snapshot: &SharedSnapshot,
10144    enabled: Option<bool>,
10145    reason: RouteCloseReason,
10146    drain_timeout: Duration,
10147) -> Result<StopNotice, SuperviseError> {
10148    let Some(forwarding) = runtime.forwarding.as_ref() else {
10149        return Ok(StopNotice::NotSent);
10150    };
10151
10152    begin_forwarding_drain_with(
10153        forwarding,
10154        ForwardingDrainContext {
10155            spec,
10156            runtime,
10157            registry,
10158            scope: DrainScope::Active,
10159        },
10160        snapshot,
10161        enabled,
10162        reason,
10163        drain_timeout,
10164    )
10165    .await
10166}
10167
10168async fn begin_forwarding_drain_with(
10169    forwarding: &ForwardingTable,
10170    context: ForwardingDrainContext<'_>,
10171    snapshot: &SharedSnapshot,
10172    enabled: Option<bool>,
10173    reason: RouteCloseReason,
10174    drain_timeout: Duration,
10175) -> Result<StopNotice, SuperviseError> {
10176    let ForwardingDrainContext {
10177        spec,
10178        runtime,
10179        registry,
10180        scope,
10181    } = context;
10182    debug_assert_ne!(reason, RouteCloseReason::Crash);
10183    let terminal = matches!(reason, RouteCloseReason::Disable);
10184    let drain_started_at = Instant::now();
10185    let drain_deadline = drain_started_at + drain_timeout;
10186    let deadline_ms =
10187        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10188    let busy_gauges = match scope {
10189        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10190        DrainScope::Endpoint(endpoint) => {
10191            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10192        }
10193    };
10194
10195    // Admission gate first: route.open/commit and route REQUEST admission are closed
10196    // before the first quiescence check, so the outstanding count can only fall.
10197    let gate_started = Instant::now();
10198    let drain_target = match scope {
10199        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10200        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10201    }
10202    .map_err(SuperviseError::Forwarding)?;
10203    // The instant admission closed, and how long taking the forwarding write
10204    // lock to close it took. The timeout line reports only the quiescence
10205    // wait, so without this a drain that started late looked like one that
10206    // started on time.
10207    info!(
10208        module_id = %spec.module_id,
10209        ?reason,
10210        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10211        connected = drain_target.is_some(),
10212        "module drain began; route admission closed"
10213    );
10214    if scope == DrainScope::Active {
10215        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10216            state.state = ModuleState::Draining;
10217            state.draining_to_replace =
10218                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10219            if let Some(enabled) = enabled {
10220                state.enabled = enabled;
10221            }
10222        })?;
10223    }
10224
10225    let Some(target) = drain_target.as_ref() else {
10226        // Nothing was sent: the module has no registered connection to carry
10227        // `module.draining` or a GOODBYE. The caller must not assume the child
10228        // was asked to stop.
10229        return Ok(StopNotice::NoConnection);
10230    };
10231    {
10232        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10233        #[cfg(feature = "test-support")]
10234        forwarding.wait_before_route_closing_for_test().await;
10235        let routes = forwarding
10236            .endpoint_routes(target.endpoint)
10237            .map_err(SuperviseError::Forwarding)?;
10238        let routes_notified = routes.len();
10239        crate::control::send_route_control_pushes(
10240            forwarding,
10241            routes.clone(),
10242            ClientControlPush::RouteClosing {
10243                module_id: spec.module_id.clone(),
10244                channels: Vec::new(),
10245                reason,
10246            },
10247        );
10248        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10249
10250        // `route.closing` was just sent above: from here on every return path,
10251        // including an early one, MUST send `route.closed` before propagating
10252        // anything else. A client holds `closing` as a promise that a verdict is
10253        // coming; leaving early without `closed` strands it waiting forever, since
10254        // `closing` carries no timeout of its own.
10255        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10256        let wait_result = wait_for_forwarding_quiescence(
10257            forwarding,
10258            &spec.module_id,
10259            runtime,
10260            target.endpoint,
10261            drain_deadline,
10262            &busy_gauges,
10263            scope,
10264            &mut gauge_trace,
10265        )
10266        .await;
10267        let drained = drained_after_quiescence_wait(&wait_result);
10268        if let Err(err) = &wait_result {
10269            error!(
10270                module_id = %spec.module_id,
10271                ?reason,
10272                error = %err,
10273                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10274            );
10275        } else if !drained {
10276            // Read before teardown releases the routes.
10277            let holdouts = forwarding
10278                .endpoint_drain_holdouts(target.endpoint)
10279                .unwrap_or_default();
10280            warn_drain_timed_out(
10281                &spec.module_id,
10282                drain_timeout,
10283                reason,
10284                &holdouts,
10285                &gauge_trace,
10286            );
10287        }
10288        crate::control::send_route_control_pushes(
10289            forwarding,
10290            routes,
10291            ClientControlPush::RouteClosed {
10292                module_id: spec.module_id.clone(),
10293                channels: Vec::new(),
10294                reason,
10295                drained,
10296                abandoned: target.abandoned_bindings.len() as u32,
10297                excluded_subscriptions: target.excluded_subscriptions,
10298                terminal: Some(terminal),
10299            },
10300        );
10301        wait_result?;
10302
10303        // `route.closed` has now been sent unconditionally above. From here the
10304        // remaining steps are cleanup (route + module GOODBYE) rather than a
10305        // promise the client is waiting on, but a lock-poisoned
10306        // `release_module_endpoint_routes` would otherwise skip the module
10307        // GOODBYE silently too -- send it before propagating the error.
10308        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10309            Ok(routes) => routes,
10310            Err(err) => {
10311                warn!(
10312                    module_id = %spec.module_id,
10313                    ?reason,
10314                    error = %err,
10315                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10316                );
10317                send_module_goodbye(&spec.module_id, forwarding, target);
10318                return Err(SuperviseError::Forwarding(err));
10319            }
10320        };
10321        let route_goodbye_count = released_routes.len();
10322        send_route_goodbyes(forwarding, released_routes);
10323        send_module_goodbye(&spec.module_id, forwarding, target);
10324
10325        // The drain's happy path was previously silent: every emission above is
10326        // best-effort with only its failure arm logged, so "were consumers told"
10327        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10328        // hang where the open question was exactly whether teardown notice went
10329        // out). One summary line makes that class decidable in one grep.
10330        info!(
10331            module_id = %spec.module_id,
10332            ?reason,
10333            routes_notified,
10334            route_goodbyes = route_goodbye_count,
10335            abandoned_reservations = target.abandoned_bindings.len(),
10336            excluded_subscriptions = target.excluded_subscriptions,
10337            drained,
10338            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10339        );
10340    }
10341
10342    Ok(StopNotice::SentOverConnection)
10343}
10344
10345/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10346/// the only slot a plain (non-swap) spawn can register into.
10347async fn wait_for_registration_after_reload(
10348    registry: &Registry,
10349    module_id: &str,
10350    snapshot: &SharedSnapshot,
10351    child: &mut SupervisedChild,
10352    wait: Duration,
10353) -> Result<RegistrationWaitOutcome, SuperviseError> {
10354    wait_for_slot_registration(
10355        registry,
10356        crate::registry::RegistrationSlot::Active(module_id),
10357        module_id,
10358        snapshot,
10359        child,
10360        wait,
10361    )
10362    .await
10363}
10364
10365/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10366///
10367/// Keyed on the slot rather than the bare module id because during a swap the
10368/// id's active slot is already held by the incumbent: an id-keyed wait would
10369/// report the incumbent's registration as the candidate's and a candidate that
10370/// never registers would look registered. A swap candidate waits on
10371/// `crate::registry::RegistrationSlot::Candidate`.
10372async fn wait_for_slot_registration(
10373    registry: &Registry,
10374    slot: crate::registry::RegistrationSlot<'_>,
10375    module_id: &str,
10376    snapshot: &SharedSnapshot,
10377    child: &mut SupervisedChild,
10378    wait: Duration,
10379) -> Result<RegistrationWaitOutcome, SuperviseError> {
10380    let deadline = Instant::now() + wait;
10381    loop {
10382        if registry
10383            .registration(slot)
10384            .map_err(SuperviseError::Registry)?
10385            .is_some()
10386        {
10387            return Ok(RegistrationWaitOutcome::Registered);
10388        }
10389
10390        let now = Instant::now();
10391        if now >= deadline {
10392            return Ok(RegistrationWaitOutcome::TimedOut);
10393        }
10394        let remaining = deadline.saturating_duration_since(now);
10395        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10396
10397        tokio::select! {
10398            wait_result = child.wait() => {
10399                let status = wait_result.map_err(|source| SuperviseError::Wait {
10400                    module_id: module_id.to_string(),
10401                    source,
10402                })?;
10403                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10404                    snapshot,
10405                    child,
10406                    &status,
10407                )));
10408            }
10409            _ = sleep(poll) => {}
10410        }
10411    }
10412}
10413
10414fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10415    // A replacement process that exits before HELLO did not provide service, even
10416    // if it used status 0. Count it against the restart cap as a new-binary failure.
10417    if exit_report.kind != ExitKind::DeliberateSeverance {
10418        exit_report.kind = ExitKind::Crash;
10419    }
10420    exit_report
10421}
10422
10423async fn handle_reload_child_registration_failure(
10424    spec: &ModuleSpec,
10425    runtime: &SupervisorRuntimeConfig,
10426    registry: &Registry,
10427    process_liveness: &SupervisorProcessLiveness,
10428    snapshot: &SharedSnapshot,
10429    _child: &mut Option<SupervisedChild>,
10430    failure: ReloadRegistrationFailure,
10431) -> Result<(), SuperviseError> {
10432    let ReloadRegistrationFailure {
10433        exit_report,
10434        reason,
10435    } = failure;
10436    match on_child_exit(
10437        spec,
10438        runtime.restart_policy,
10439        registry,
10440        snapshot,
10441        &runtime.terminal_ring,
10442        &runtime.spawn_events,
10443        &runtime.child_roster,
10444        exit_report,
10445    )
10446    .await
10447    {
10448        NextAction::Stop {
10449            registration_released,
10450        } => {
10451            if registration_released {
10452                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10453            }
10454        }
10455        NextAction::Restart { schedule } => {
10456            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10457                schedule.delay
10458            });
10459            if let Some(schedule) = schedule {
10460                log_crash_respawn(&spec.module_id, schedule);
10461            }
10462            schedule_respawn(
10463                runtime,
10464                snapshot,
10465                &spec.module_id,
10466                delay,
10467                RespawnKind::Spawn,
10468            )?;
10469        }
10470    }
10471    Err(SuperviseError::ReloadFailed {
10472        module_id: spec.module_id.clone(),
10473        reason,
10474    })
10475}
10476
10477async fn handle_reload_spawn_failure(
10478    spec: &ModuleSpec,
10479    runtime: &SupervisorRuntimeConfig,
10480    process_liveness: &SupervisorProcessLiveness,
10481    snapshot: &SharedSnapshot,
10482    _child: &mut Option<SupervisedChild>,
10483    reason: String,
10484) -> Result<(), SuperviseError> {
10485    let now = Instant::now();
10486    let mut schedule = None;
10487    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10488        clear_current_process_facts(state);
10489        if state.enabled {
10490            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10491            state.state = if schedule.is_some() {
10492                ModuleState::Restarting
10493            } else {
10494                ModuleState::Failed
10495            };
10496        } else {
10497            state.state = ModuleState::Disabled;
10498        }
10499    })?;
10500    if let Some(schedule) = schedule {
10501        schedule_respawn(
10502            runtime,
10503            snapshot,
10504            &spec.module_id,
10505            schedule.delay,
10506            RespawnKind::Spawn,
10507        )?;
10508    } else {
10509        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10510    }
10511    Err(SuperviseError::ReloadFailed {
10512        module_id: spec.module_id.clone(),
10513        reason,
10514    })
10515}
10516
10517fn control_flags() -> Flags {
10518    Flags::new(false, Priority::Passive, false)
10519}
10520
10521#[allow(clippy::too_many_arguments)]
10522async fn drain_optional_child(
10523    module_id: &str,
10524    protocol: ModuleProtocol,
10525    stop_notice: StopNotice,
10526    registry: &Registry,
10527    forwarding: Option<&ForwardingTable>,
10528    snapshot: &SharedSnapshot,
10529    terminal_ring: &Arc<Mutex<TerminalRing>>,
10530    spawn_events: &SpawnEventFeed,
10531    child: &mut Option<SupervisedChild>,
10532    drain_timeout: Duration,
10533    final_state: ModuleState,
10534    enabled: Option<bool>,
10535) -> Result<(), SuperviseError> {
10536    if let Some(child) = child.take() {
10537        drain_child_to_state(
10538            module_id,
10539            protocol,
10540            stop_notice,
10541            registry,
10542            forwarding,
10543            snapshot,
10544            terminal_ring,
10545            spawn_events,
10546            child,
10547            drain_timeout,
10548            final_state,
10549            enabled,
10550        )
10551        .await
10552    } else {
10553        update_snapshot(snapshot, Some(module_id), |state| {
10554            state.state = final_state;
10555            if let Some(enabled) = enabled {
10556                state.enabled = enabled;
10557            }
10558            clear_current_process_facts(state);
10559        })?;
10560        release_dead_registration(registry, forwarding, snapshot, module_id).await
10561    }
10562}
10563
10564#[allow(clippy::too_many_arguments)]
10565async fn drain_child_to_state(
10566    module_id: &str,
10567    _protocol: ModuleProtocol,
10568    stop_notice: StopNotice,
10569    registry: &Registry,
10570    forwarding: Option<&ForwardingTable>,
10571    snapshot: &SharedSnapshot,
10572    terminal_ring: &Arc<Mutex<TerminalRing>>,
10573    spawn_events: &SpawnEventFeed,
10574    mut child: SupervisedChild,
10575    drain_timeout: Duration,
10576    final_state: ModuleState,
10577    enabled: Option<bool>,
10578) -> Result<(), SuperviseError> {
10579    let protocol = child.protocol;
10580    update_snapshot(snapshot, Some(module_id), |state| {
10581        state.state = ModuleState::Draining;
10582        state.draining_to_replace = final_state == ModuleState::Restarting;
10583        if let Some(enabled) = enabled {
10584            state.enabled = enabled;
10585        }
10586    })?;
10587
10588    // The wait below is the same budget in every case; what differs is
10589    // whether anything has ASKED the child to stop before it starts. Only a
10590    // forwarding drain that reached the module's registered connection has
10591    // (`module.draining`, then a module GOODBYE). Every other child was told
10592    // nothing: a `protocol: "none"` module, which never registers; a subc
10593    // module spawned moments ago that has not sent HELLO yet; or a stop that
10594    // runs no forwarding drain. Without a signal the budget is only a delay
10595    // in front of SIGKILL -- and the not-yet-registered child is the worst
10596    // case, because it registers into a module that is already draining,
10597    // is never told, and is killed while healthy.
10598    if stop_notice != StopNotice::SentOverConnection {
10599        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10600            info!(
10601                module_id,
10602                pid = child.pid,
10603                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10604                "module has no connection yet; requesting stop by signal"
10605            );
10606        }
10607        request_graceful_stop(module_id, &child);
10608    }
10609
10610    let exit_report = match timeout(drain_timeout, child.wait()).await {
10611        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10612        Ok(Err(source)) => {
10613            fail_after_wait_error(
10614                module_id,
10615                snapshot,
10616                terminal_ring,
10617                spawn_events,
10618                format!("failed to wait during drain: {source}"),
10619            );
10620            return Err(SuperviseError::Wait {
10621                module_id: module_id.to_string(),
10622                source,
10623            });
10624        }
10625        Err(_) => {
10626            // Mirror the sibling arm above: state is already `Draining`, and an
10627            // error propagated from here would strand it there -- a state
10628            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10629            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10630            // `Failed` before `?` keeps the module operator-visible and
10631            // revivable. Trigger is an ESRCH race (process exits between the
10632            // drain timeout firing and the kill) or a post-kill wait failure
10633            // (issue #34).
10634            //
10635            // Logged because the kill is otherwise visible only as signal 9 in
10636            // the terminal ring, and the budget it follows can be long enough
10637            // that consumers see a stretch of refusals with no stated cause.
10638            warn!(
10639                module_id,
10640                pid = child.pid,
10641                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10642                reason = ?final_state,
10643                ?stop_notice,
10644                "drain budget expired before the module exited; killing it"
10645            );
10646            child.start_kill().map_err(|source| {
10647                fail_after_wait_error(
10648                    module_id,
10649                    snapshot,
10650                    terminal_ring,
10651                    spawn_events,
10652                    format!("failed to kill during drain: {source}"),
10653                );
10654                SuperviseError::Kill {
10655                    module_id: module_id.to_string(),
10656                    source,
10657                }
10658            })?;
10659            let status = child.wait().await.map_err(|source| {
10660                fail_after_wait_error(
10661                    module_id,
10662                    snapshot,
10663                    terminal_ring,
10664                    spawn_events,
10665                    format!("failed to wait after drain kill: {source}"),
10666                );
10667                SuperviseError::Wait {
10668                    module_id: module_id.to_string(),
10669                    source,
10670                }
10671            })?;
10672            classify_reaped_child_exit(snapshot, &child, &status)
10673        }
10674    };
10675
10676    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10677    publish_after_terminal_record(
10678        module_id,
10679        || {
10680            record_terminal_with_detail(
10681                module_id,
10682                terminal_ring,
10683                spawn_events,
10684                &exit_report,
10685                terminal_disposition(final_state),
10686                detail,
10687            )
10688        },
10689        || {
10690            update_snapshot(snapshot, Some(module_id), |state| {
10691                state.state = final_state;
10692                if let Some(enabled) = enabled {
10693                    state.enabled = enabled;
10694                }
10695                clear_current_process_facts(state);
10696                state.last_exit = Some(exit_report.clone());
10697                if exit_report.kind == ExitKind::DeliberateSeverance {
10698                    state.lifetime_restarts += 1;
10699                }
10700            })
10701        },
10702    )?;
10703    child.drain_stderr(module_id).await;
10704
10705    release_dead_registration(registry, forwarding, snapshot, module_id).await
10706}
10707
10708/// Ask a child that nothing else has asked to stop, by signal.
10709///
10710/// A registered subc module is asked over its own connection: the drain sends
10711/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10712/// module GOODBYE, and the module stops itself. A module that speaks no subc
10713/// wire receives none of that, and neither does a subc module that has not
10714/// registered yet, so for them the drain budget would be pure delay in front of
10715/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10716/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10717/// into a recovery on the next start.
10718///
10719/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10720/// rule rather than an optimisation: that module's graceful stop is already
10721/// running by the time its child is drained, and a signal would race it.
10722///
10723/// Best-effort by construction. A child that has already exited is the ordinary
10724/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10725/// failure is logged at debug and the wait-then-kill below still decides the
10726/// outcome.
10727#[cfg(unix)]
10728fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10729    let Some(pid) = child
10730        .id()
10731        .and_then(|pid| i32::try_from(pid).ok())
10732        .and_then(rustix::process::Pid::from_raw)
10733    else {
10734        debug!(
10735            module_id,
10736            "no pid to signal for teardown; falling through to the drain wait"
10737        );
10738        return;
10739    };
10740    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10741        Ok(()) => debug!(
10742            module_id,
10743            "sent SIGTERM to a module nothing else asked to stop"
10744        ),
10745        Err(err) => debug!(
10746            module_id,
10747            error = %err,
10748            "SIGTERM to module failed; the drain wait and kill still apply"
10749        ),
10750    }
10751}
10752
10753/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10754/// Windows does offer need cooperation this supervisor cannot assume: a console
10755/// control event requires sharing a console with the child, and `WM_CLOSE`
10756/// requires the child to pump a message loop. A supervised server process does
10757/// neither, so there is nothing to send and teardown is the wait followed by the
10758/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10759/// the thing `protocol: "none"` exists to avoid.
10760#[cfg(not(unix))]
10761fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10762    debug!(
10763        module_id,
10764        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10765    );
10766}
10767
10768fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10769    match final_state {
10770        ModuleState::Stopped => TerminalDisposition::Stopped,
10771        ModuleState::Disabled => TerminalDisposition::Disabled,
10772        ModuleState::Restarting => TerminalDisposition::Restarting,
10773        ModuleState::Failed => TerminalDisposition::Failed,
10774        ModuleState::Starting
10775        | ModuleState::Running
10776        | ModuleState::Unresponsive
10777        | ModuleState::Draining => {
10778            unreachable!("terminal exits only finish in terminal or restarting states")
10779        }
10780    }
10781}
10782
10783/// Release a reaped child's registration before allowing another spawn.
10784///
10785/// EOF is not a process-lifetime signal: an inherited socket can stay open
10786/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10787/// reading EOF. After the normal release grace, request connection close (which
10788/// cancels both reads and dispatch), then allow one more release grace for the
10789/// connection guard's forwarding cleanup. Never evict a different connection.
10790async fn release_dead_registration(
10791    registry: &Registry,
10792    forwarding: Option<&ForwardingTable>,
10793    snapshot: &SharedSnapshot,
10794    module_id: &str,
10795) -> Result<(), SuperviseError> {
10796    let result = async {
10797        let registration = registry
10798            .get_module(module_id)
10799            .map_err(SuperviseError::Registry)?;
10800        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10801            Ok(()) => return Ok(()),
10802            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10803            Err(err) => return Err(err),
10804        }
10805        let pid = lock_snapshot(snapshot)?.reaped_pid;
10806        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10807            warn!(
10808                module_id,
10809                pid,
10810                connection_id = registration.connection_id.get(),
10811                "reaped module registration outlived release grace; closing dead connection"
10812            );
10813            forwarding.request_connection_close(
10814                registration.connection_id,
10815                CloseReason::new(
10816                    "supervised_process_reaped",
10817                    format!("module '{module_id}' pid {pid} exited"),
10818                ),
10819            );
10820            wait_for_slot_registration_release(
10821                registry,
10822                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10823                REGISTRY_RELEASE_TIMEOUT,
10824            )
10825            .await?;
10826        }
10827        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10828    }
10829    .await;
10830    if let Err(err) = &result {
10831        fail_snapshot(snapshot, Some(module_id), None);
10832        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10833    }
10834    result
10835}
10836
10837/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10838/// plain stop or restart waits for before it spawns a replacement.
10839async fn wait_for_registration_release(
10840    registry: &Registry,
10841    module_id: &str,
10842    wait: Duration,
10843) -> Result<(), SuperviseError> {
10844    wait_for_slot_registration_release(
10845        registry,
10846        crate::registry::RegistrationSlot::Active(module_id),
10847        wait,
10848    )
10849    .await
10850}
10851
10852/// Wait for the registration in `slot` to go away.
10853///
10854/// Keyed on the slot rather than the bare module id because a successful swap
10855/// never empties the id's active slot (the promoted candidate is in it), so an
10856/// id-keyed wait for the incumbent's release would always time out. Draining a
10857/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10858/// incumbent's connection instead.
10859async fn wait_for_slot_registration_release(
10860    registry: &Registry,
10861    slot: crate::registry::RegistrationSlot<'_>,
10862    wait: Duration,
10863) -> Result<(), SuperviseError> {
10864    let deadline = Instant::now() + wait;
10865    let mut release_events = registration_release_events().subscribe();
10866    let still_active = |registration: &crate::registry::ModuleRegistration| {
10867        SuperviseError::RegistrationStillActive {
10868            module_id: registration.manifest.module_id.clone(),
10869            waited: wait,
10870        }
10871    };
10872    loop {
10873        let _observed_generation = *release_events.borrow_and_update();
10874        let Some(registration) = registry
10875            .registration(slot)
10876            .map_err(SuperviseError::Registry)?
10877        else {
10878            return Ok(());
10879        };
10880
10881        let now = Instant::now();
10882        if now >= deadline {
10883            return Err(still_active(&registration));
10884        }
10885
10886        let remaining = deadline.saturating_duration_since(now);
10887        match timeout(remaining, release_events.changed()).await {
10888            Ok(Ok(())) | Ok(Err(_)) => {}
10889            Err(_) => return Err(still_active(&registration)),
10890        }
10891    }
10892}
10893
10894#[cfg(test)]
10895mod slot_registration_wait_tests {
10896    use super::*;
10897    use crate::registry::{ConnectionId, RegistrationSlot};
10898    use subc_protocol::manifest::ModuleManifest;
10899
10900    #[tokio::test]
10901    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10902        let registry = Arc::new(Registry::default());
10903        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10904        let runtime = supervisor.runtime_config();
10905        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10906        let spec = ModuleSpec {
10907            module_id: "enable-stale-registration".to_string(),
10908            program: PathBuf::from("/missing/enable-retry-test"),
10909            args: Vec::new(),
10910            env: Vec::new(),
10911            reserved: false,
10912            reserved_prefixes: Vec::new(),
10913            protocol: ModuleProtocol::Subc,
10914            overlap: Default::default(),
10915        };
10916        let connection = ConnectionId::new(90);
10917        registry
10918            .register_with_control_ops(
10919                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10920                1,
10921                connection,
10922                Vec::new(),
10923            )
10924            .unwrap();
10925        let mut child = None;
10926        let err = set_child_enabled(
10927            &spec,
10928            &runtime,
10929            &registry,
10930            &supervisor.process_liveness,
10931            &snapshot,
10932            &mut child,
10933            true,
10934        )
10935        .await
10936        .unwrap_err();
10937        assert!(matches!(
10938            err,
10939            SuperviseError::RegistrationStillActive { .. }
10940        ));
10941        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10942        assert!(child.is_none());
10943        registry.deregister_connection(connection).unwrap();
10944        let err = set_child_enabled(
10945            &spec,
10946            &runtime,
10947            &registry,
10948            &supervisor.process_liveness,
10949            &snapshot,
10950            &mut child,
10951            true,
10952        )
10953        .await
10954        .unwrap_err();
10955        assert!(
10956            matches!(err, SuperviseError::Spawn { .. }),
10957            "second enable must attempt a spawn: {err}"
10958        );
10959        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10960    }
10961
10962    const INCUMBENT: u64 = 1;
10963    const CANDIDATE: u64 = 2;
10964
10965    fn swapped_registry() -> Arc<Registry> {
10966        let registry = Arc::new(Registry::default());
10967        let manifest = ModuleManifest::builder("m", "0.1.0").build();
10968        registry
10969            .register_with_control_ops(
10970                manifest.clone(),
10971                1,
10972                ConnectionId::new(INCUMBENT),
10973                Vec::new(),
10974            )
10975            .unwrap();
10976        registry
10977            .register_candidate_with_control_ops(
10978                manifest,
10979                1,
10980                ConnectionId::new(CANDIDATE),
10981                Vec::new(),
10982            )
10983            .unwrap();
10984        registry
10985    }
10986
10987    /// After a promotion the id's active slot is held by the new process, so an
10988    /// id-keyed wait for the incumbent's release can never succeed; the
10989    /// connection-keyed wait completes as soon as the incumbent deregisters.
10990    #[tokio::test]
10991    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10992        let registry = swapped_registry();
10993        registry.promote_candidate("m").unwrap().unwrap();
10994
10995        assert!(matches!(
10996            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
10997            Err(SuperviseError::RegistrationStillActive { .. })
10998        ));
10999
11000        // Still held while the incumbent's connection has not deregistered.
11001        assert!(matches!(
11002            wait_for_slot_registration_release(
11003                &registry,
11004                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11005                Duration::from_millis(50),
11006            )
11007            .await,
11008            Err(SuperviseError::RegistrationStillActive { .. })
11009        ));
11010
11011        let releaser = Arc::clone(&registry);
11012        let release = tokio::spawn(async move {
11013            sleep(Duration::from_millis(20)).await;
11014            releaser
11015                .deregister_connection(ConnectionId::new(INCUMBENT))
11016                .unwrap();
11017            notify_registration_release();
11018        });
11019        wait_for_slot_registration_release(
11020            &registry,
11021            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11022            Duration::from_secs(5),
11023        )
11024        .await
11025        .expect("the incumbent's own registration is released");
11026        release.await.unwrap();
11027        assert!(registry.get_module("m").unwrap().is_some());
11028    }
11029
11030    /// The candidate slot is waited on separately from the active slot: the
11031    /// incumbent's registration neither holds up nor stands in for it.
11032    #[tokio::test]
11033    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11034        let registry = swapped_registry();
11035        assert!(matches!(
11036            wait_for_slot_registration_release(
11037                &registry,
11038                RegistrationSlot::Candidate("m"),
11039                Duration::from_millis(50),
11040            )
11041            .await,
11042            Err(SuperviseError::RegistrationStillActive { .. })
11043        ));
11044        registry
11045            .deregister_connection(ConnectionId::new(CANDIDATE))
11046            .unwrap();
11047        wait_for_slot_registration_release(
11048            &registry,
11049            RegistrationSlot::Candidate("m"),
11050            Duration::from_millis(50),
11051        )
11052        .await
11053        .expect("a candidate slot with no candidate is released");
11054        assert!(registry
11055            .registration(RegistrationSlot::Active("m"))
11056            .unwrap()
11057            .is_some());
11058    }
11059}
11060
11061fn classify_exit(status: &ExitStatus) -> ExitReport {
11062    ExitReport {
11063        kind: if status.success() {
11064            ExitKind::Clean
11065        } else {
11066            ExitKind::Crash
11067        },
11068        code: status.code(),
11069        signal: exit_signal(status),
11070        at_ms: unix_ms_now(),
11071    }
11072}
11073
11074/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11075/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11076/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11077/// disposition still must be `Failed` so the terminal ring is not silently missing
11078/// an entry, matching what `fail_snapshot` records for this same arm.
11079fn wait_error_exit_report() -> ExitReport {
11080    ExitReport {
11081        kind: ExitKind::Crash,
11082        code: None,
11083        signal: None,
11084        at_ms: unix_ms_now(),
11085    }
11086}
11087
11088#[cfg(unix)]
11089fn exit_signal(status: &ExitStatus) -> Option<i32> {
11090    use std::os::unix::process::ExitStatusExt;
11091
11092    status.signal()
11093}
11094
11095#[cfg(not(unix))]
11096fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11097    None
11098}
11099
11100/// Give an operator-touched module its full crash budget back.
11101///
11102/// Named for the counter it used to zero; it now empties the in-window ring,
11103/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11104/// ledger of what happened survives every operator action.
11105fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11106    update_snapshot(snapshot, Some(module_id), |state| {
11107        state.clear_crash_restarts();
11108    })
11109}
11110
11111fn set_running(
11112    snapshot: &SharedSnapshot,
11113    child: &SupervisedChild,
11114    module_id: &str,
11115    spawn_events: &SpawnEventFeed,
11116) -> Result<(), SuperviseError> {
11117    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11118        module_id: Some(module_id.to_string()),
11119    })?;
11120    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11121    if std::mem::take(&mut state.coalesced_restart_pending) {
11122        let generation = state.spawn_generation;
11123        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11124    }
11125    state.drain_disposition_detail = None;
11126    state.spawn_failure = None;
11127    state.waiting_for_previous_pid = None;
11128    state.waiting_for_previous_detail = None;
11129    // Every caller of this is a plain spawn, which always uses the primary key;
11130    // a promoted swap candidate sets the flag itself after this returns.
11131    state.in_alternate_slot = false;
11132    state.configuration_updated_since_spawn = false;
11133    state.spawned_protocol = Some(child.protocol);
11134    state.state = ModuleState::Running;
11135    state.enabled = true;
11136    state.process_alive = true;
11137    state.pid = child.id();
11138    #[cfg(windows)]
11139    {
11140        state.pipe_consumed = child
11141            .nonce_delivery
11142            .as_ref()
11143            .map(|delivery| delivery.consumed());
11144    }
11145    #[cfg(target_os = "macos")]
11146    {
11147        state.report_ready = Some(Arc::clone(&child.report_ready));
11148    }
11149    state.spawned_at_ms = Some(child.spawned_at_ms);
11150    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11151    state.spawned_from = Some(child.spawned_from.clone());
11152    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11153    state.spawned_file_identity = child.spawned_file_identity.clone();
11154    state.process_start_time = child.process_start_time;
11155    Ok(())
11156}
11157
11158fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11159    state.waiting_for_previous_pid = None;
11160    state.waiting_for_previous_detail = None;
11161    state.process_alive = false;
11162    state.spawned_protocol = None;
11163    state.pid = None;
11164    #[cfg(windows)]
11165    {
11166        state.pipe_consumed = None;
11167    }
11168    #[cfg(target_os = "macos")]
11169    {
11170        state.report_ready = None;
11171    }
11172    state.spawned_at_ms = None;
11173    state.exec_outcome = None;
11174    state.spawned_from = None;
11175    state.launch_spec_digest = None;
11176    state.spawned_file_identity = None;
11177    state.process_start_time = None;
11178    state.deliberate_severance = None;
11179}
11180
11181#[cfg(test)]
11182fn record_deliberate_severance(
11183    snapshot: &SharedSnapshot,
11184    identity: ProcessIdentity,
11185) -> Result<(), SuperviseError> {
11186    update_snapshot(snapshot, None, |state| {
11187        state.deliberate_severance = Some(identity);
11188    })
11189}
11190
11191fn apply_deliberate_severance_marker(
11192    snapshot: &SharedSnapshot,
11193    exited_identity: Option<ProcessIdentity>,
11194    mut exit_report: ExitReport,
11195) -> ExitReport {
11196    let marker = lock_snapshot(snapshot)
11197        .ok()
11198        .and_then(|mut state| state.deliberate_severance.take());
11199    if marker.is_some() && marker == exited_identity {
11200        exit_report.kind = ExitKind::DeliberateSeverance;
11201    }
11202    exit_report
11203}
11204
11205fn classify_reaped_child_exit(
11206    snapshot: &SharedSnapshot,
11207    child: &SupervisedChild,
11208    status: &ExitStatus,
11209) -> ExitReport {
11210    let _ = update_snapshot(snapshot, None, |state| {
11211        state.reaped_pid = Some(child.pid);
11212        state.spawn_failure = child.spawn_failure.clone();
11213    });
11214    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11215}
11216
11217fn fail_snapshot(
11218    snapshot: &SharedSnapshot,
11219    module_id: Option<&str>,
11220    last_exit: Option<ExitReport>,
11221) {
11222    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11223        state.state = ModuleState::Failed;
11224        clear_current_process_facts(state);
11225        if let Some(last_exit) = last_exit {
11226            state.last_exit = Some(last_exit);
11227        }
11228    }) {
11229        error!(error = %err, "failed to mark supervisor state failed");
11230    }
11231}
11232
11233fn update_snapshot(
11234    snapshot: &SharedSnapshot,
11235    module_id: Option<&str>,
11236    update: impl FnOnce(&mut SupervisorSnapshot),
11237) -> Result<(), SuperviseError> {
11238    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11239        module_id: module_id.map(ToOwned::to_owned),
11240    })?;
11241    update(&mut state);
11242    Ok(())
11243}
11244
11245const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11246
11247fn lock_snapshot_for_control<'a>(
11248    snapshot: &'a SharedSnapshot,
11249    module_id: &str,
11250    caller: &'static str,
11251) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11252    let started_at = Instant::now();
11253    let guard = lock_snapshot(snapshot)?;
11254    let waited = started_at.elapsed();
11255    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11256        warn!(
11257            module_id = %module_id,
11258            waited_ms = waited.as_millis() as u64,
11259            caller = %caller,
11260            "slow snapshot lock"
11261        );
11262    }
11263    Ok(guard)
11264}
11265
11266fn lock_snapshot(
11267    snapshot: &SharedSnapshot,
11268) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11269    snapshot
11270        .lock()
11271        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11272}
11273
11274#[cfg(test)]
11275mod terminal_history_tests {
11276    use std::{
11277        path::PathBuf,
11278        sync::Arc,
11279        time::{Duration, Instant},
11280    };
11281
11282    use tokio::{sync::mpsc, time::sleep};
11283
11284    use super::{
11285        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11286        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11287        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11288        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11289        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11290        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11291        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11292        SupervisorHealthStatus, SupervisorSnapshot,
11293    };
11294    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11295    // use for their own wall-clock deadlines: crash-restart instants must be on
11296    // the same clock the production code stamps them with, which is tokio's (and
11297    // is what `start_paused` tests can move).
11298    use super::Instant as ClockInstant;
11299    use crate::{
11300        registry::Registry,
11301        terminal_ring::{TerminalRing, TerminalRingConfig},
11302    };
11303    use std::sync::Mutex;
11304    use subc_control::TerminalDisposition;
11305
11306    /// See the twin in `control.rs` for why this derives the path from
11307    /// `current_exe()` and why the existence check is here: `--lib` alone does
11308    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11309    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11310    pub(super) fn fake_aft_stub_path() -> PathBuf {
11311        let mut path = std::env::current_exe().expect("current_exe available in tests");
11312        path.pop();
11313        path.pop();
11314        path.push(if cfg!(windows) {
11315            "fake-aft-stub.exe"
11316        } else {
11317            "fake-aft-stub"
11318        });
11319        assert!(
11320            path.exists(),
11321            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11322             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11323            path.display()
11324        );
11325        path
11326    }
11327
11328    #[test]
11329    fn reserved_never_spawned_refuses_every_hello() {
11330        // The canary hole: a reserved id whose module has never spawned had NO
11331        // gate entry and admitted anyone -- the reservation protected the nonce
11332        // holder, not the NAME. Now the entry is present with no legitimate
11333        // holder and refuses all comers.
11334        let supervisor = SupervisorHandle::default();
11335        supervisor.apply_identity_configuration(&ModuleSpec {
11336            module_id: "never-spawned".to_string(),
11337            program: PathBuf::from("/usr/bin/false"),
11338            args: Vec::new(),
11339            env: Vec::new(),
11340            reserved: true,
11341            reserved_prefixes: Vec::new(),
11342            protocol: ModuleProtocol::Subc,
11343            overlap: Default::default(),
11344        });
11345        assert!(
11346            supervisor
11347                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11348                .is_some(),
11349            "forged nonce must refuse on a reserved never-spawned id"
11350        );
11351        assert!(
11352            supervisor
11353                .reserved_hello_rejection("never-spawned", None)
11354                .is_some(),
11355            "absent nonce must refuse on a reserved never-spawned id"
11356        );
11357        // And a real spawn nonce minted later admits exactly that nonce.
11358        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11359        supervisor.apply_identity_configuration(&ModuleSpec {
11360            module_id: "never-spawned".to_string(),
11361            program: PathBuf::from("/usr/bin/false"),
11362            args: Vec::new(),
11363            env: Vec::new(),
11364            reserved: true,
11365            reserved_prefixes: Vec::new(),
11366            protocol: ModuleProtocol::Subc,
11367            overlap: Default::default(),
11368        });
11369        assert!(supervisor
11370            .reserved_hello_rejection("never-spawned", Some("minted"))
11371            .is_none());
11372        assert!(supervisor
11373            .reserved_hello_rejection("never-spawned", Some("forged"))
11374            .is_some());
11375    }
11376
11377    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11378    /// happened, which is what "spent budget" looks like to every reader.
11379    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11380        let now = ClockInstant::now();
11381        for _ in 0..count {
11382            state.crash_restarts.push_back(now);
11383        }
11384    }
11385
11386    /// Age the oldest recorded restart out of `window`, standing in for the hours
11387    /// that would otherwise have to pass. Injecting the instant is the point: a
11388    /// test that slept a real window would take ten minutes and still prove less.
11389    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11390        let aged = state
11391            .crash_restarts
11392            .front()
11393            .expect("a crash restart must be recorded before it can be aged")
11394            .checked_sub(window + Duration::from_secs(1))
11395            .expect("the test clock is far enough from its origin to age an instant");
11396        state.crash_restarts[0] = aged;
11397    }
11398
11399    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11400        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11401        seed_crash_restarts(&mut state, count);
11402        state
11403    }
11404
11405    #[test]
11406    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11407        let policy = RestartPolicy::new(3, Duration::ZERO);
11408        let now = ClockInstant::now();
11409        assert!(daemon_will_restart(
11410            &mut snapshot_with_restarts(true, 2),
11411            &policy,
11412            now
11413        ));
11414        assert!(!daemon_will_restart(
11415            &mut snapshot_with_restarts(true, 3),
11416            &policy,
11417            now
11418        ));
11419        assert!(!daemon_will_restart(
11420            &mut snapshot_with_restarts(false, 0),
11421            &policy,
11422            now
11423        ));
11424    }
11425
11426    #[test]
11427    fn crash_restart_backoff_escalates_with_in_window_count() {
11428        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11429            .with_max_backoff(Duration::from_secs(30));
11430        let now = ClockInstant::now();
11431        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11432        let schedules = (0..4)
11433            .map(|_| {
11434                state
11435                    .next_crash_restart(&policy, now)
11436                    .expect("the test policy allows four crash restarts")
11437            })
11438            .collect::<Vec<_>>();
11439
11440        assert_eq!(
11441            schedules
11442                .iter()
11443                .map(|schedule| schedule.restart_in_window)
11444                .collect::<Vec<_>>(),
11445            vec![0, 1, 2, 3]
11446        );
11447        assert_eq!(
11448            schedules
11449                .iter()
11450                .map(|schedule| schedule.delay)
11451                .collect::<Vec<_>>(),
11452            vec![
11453                Duration::from_millis(100),
11454                Duration::from_secs(1),
11455                Duration::from_secs(10),
11456                Duration::from_secs(30),
11457            ]
11458        );
11459    }
11460
11461    #[test]
11462    fn crash_restart_backoff_resets_after_ring_clear() {
11463        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11464        let now = ClockInstant::now();
11465        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11466        assert_eq!(
11467            state.next_crash_restart(&policy, now).unwrap().delay,
11468            Duration::from_millis(100)
11469        );
11470        assert_eq!(
11471            state.next_crash_restart(&policy, now).unwrap().delay,
11472            Duration::from_secs(1)
11473        );
11474
11475        state.clear_crash_restarts();
11476        let schedule = state
11477            .next_crash_restart(&policy, now)
11478            .expect("a cleared ring must allow another restart");
11479        assert_eq!(schedule.restart_in_window, 0);
11480        assert_eq!(schedule.delay, Duration::from_millis(100));
11481    }
11482
11483    #[test]
11484    fn crash_restart_backoff_ignores_aged_restarts() {
11485        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11486        let now = ClockInstant::now();
11487        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11488        state
11489            .next_crash_restart(&policy, now)
11490            .expect("the first restart is allowed");
11491        state
11492            .next_crash_restart(&policy, now)
11493            .expect("the second restart is allowed");
11494        state.crash_restarts[0] = now
11495            .checked_sub(policy.window + Duration::from_secs(1))
11496            .expect("the fake clock can age a restart past the window");
11497
11498        let schedule = state
11499            .next_crash_restart(&policy, now)
11500            .expect("an aged restart must release its slot");
11501        assert_eq!(schedule.restart_in_window, 1);
11502        assert_eq!(schedule.delay, Duration::from_secs(1));
11503        assert_eq!(state.crash_restarts.len(), 2);
11504    }
11505
11506    /// The budget is a rate: the same three spent restarts refuse a respawn
11507    /// while they are recent and allow one once they have aged past the window.
11508    /// Nothing about the module changed in between, which is the whole point.
11509    #[test]
11510    fn a_budget_spent_before_the_window_no_longer_refuses() {
11511        let policy = RestartPolicy::new(3, Duration::ZERO);
11512        let mut state = snapshot_with_restarts(true, 3);
11513        let now = ClockInstant::now();
11514        assert!(!daemon_will_restart(&mut state, &policy, now));
11515
11516        assert!(daemon_will_restart(
11517            &mut state,
11518            &policy,
11519            now + policy.window + Duration::from_secs(1)
11520        ));
11521        assert!(
11522            state.crash_restarts.is_empty(),
11523            "reading the budget must drop the instants that left the window"
11524        );
11525    }
11526
11527    fn module_with_recovery_snapshot(
11528        state: ModuleState,
11529        enabled: bool,
11530        restart_count: u32,
11531    ) -> SupervisedModule {
11532        let registry = Arc::new(Registry::default());
11533        let supervisor =
11534            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11535        let runtime = supervisor.runtime_config();
11536        let spec = ModuleSpec {
11537            module_id: "recovery-snapshot".to_string(),
11538            program: fake_aft_stub_path(),
11539            args: Vec::new(),
11540            env: Vec::new(),
11541            reserved: false,
11542            reserved_prefixes: Vec::new(),
11543            protocol: ModuleProtocol::Subc,
11544            overlap: Default::default(),
11545        };
11546        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11547        seed_crash_restarts(&mut snapshot, restart_count);
11548        // These tests read synthetic snapshots. A real child and monitor would
11549        // race those reads by replacing the requested state during startup.
11550        let (commands, _rx) = mpsc::channel(4);
11551        SupervisedModule {
11552            inner: Arc::new(SupervisedModuleInner {
11553                module_id: spec.module_id.clone(),
11554                registry,
11555                snapshot: Arc::new(Mutex::new(snapshot)),
11556                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11557                    spec,
11558                    health: runtime.health,
11559                })),
11560                stderr_ring: runtime.stderr_ring,
11561                terminal_ring: runtime.terminal_ring,
11562                commands,
11563                monitor: Mutex::new(None),
11564                restart_policy: runtime.restart_policy,
11565                effective_drain_timeout: runtime.effective_drain_timeout,
11566                provenance_probe: supervisor.provenance_probe.clone(),
11567            }),
11568        }
11569    }
11570
11571    #[cfg(target_os = "linux")]
11572    #[tokio::test]
11573    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11574        let supervisor =
11575            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11576                .with_cgroup_placement(None);
11577        let result = supervisor.spawn(ModuleSpec {
11578            module_id: "no-cgroup-placement".to_string(),
11579            program: fake_aft_stub_path(),
11580            args: Vec::new(),
11581            env: Vec::new(),
11582            reserved: false,
11583            reserved_prefixes: Vec::new(),
11584            protocol: ModuleProtocol::Subc,
11585            overlap: Default::default(),
11586        });
11587
11588        assert!(
11589            result.is_ok(),
11590            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11591        );
11592    }
11593
11594    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11595    async fn undecided_snapshot_uses_shared_restart_predicate() {
11596        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11597            .will_recover_after_connection_loss()
11598            .unwrap());
11599        assert!(
11600            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11601                .will_recover_after_connection_loss()
11602                .unwrap()
11603        );
11604    }
11605
11606    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11607    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11608        assert!(
11609            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11610                .will_recover_after_connection_loss()
11611                .unwrap()
11612        );
11613    }
11614
11615    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11616    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11617        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11618            .will_recover_after_connection_loss()
11619            .unwrap());
11620        assert!(
11621            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11622                .will_recover_after_connection_loss()
11623                .unwrap()
11624        );
11625    }
11626
11627    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11628    async fn warming_snapshot_is_limited_to_startup_phases() {
11629        for state in [
11630            ModuleState::Starting,
11631            ModuleState::Running,
11632            ModuleState::Restarting,
11633        ] {
11634            assert!(
11635                module_with_recovery_snapshot(state, true, 0)
11636                    .is_warming()
11637                    .unwrap(),
11638                "{state:?} should be warming"
11639            );
11640        }
11641        for state in [
11642            ModuleState::Unresponsive,
11643            ModuleState::Draining,
11644            ModuleState::Stopped,
11645            ModuleState::Failed,
11646            ModuleState::Disabled,
11647        ] {
11648            assert!(
11649                !module_with_recovery_snapshot(state, true, 0)
11650                    .is_warming()
11651                    .unwrap(),
11652                "{state:?} should not be warming"
11653            );
11654        }
11655    }
11656
11657    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11658    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11659        let registry = Arc::new(Registry::default());
11660        let supervisor =
11661            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11662        let module = supervisor
11663            .spawn(ModuleSpec {
11664                module_id: "terminal-history".to_string(),
11665                program: fake_aft_stub_path(),
11666                args: Vec::new(),
11667                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11668                reserved: false,
11669                reserved_prefixes: Vec::new(),
11670                protocol: ModuleProtocol::Subc,
11671                overlap: Default::default(),
11672            })
11673            .unwrap();
11674
11675        let deadline = Instant::now() + Duration::from_secs(5);
11676        loop {
11677            let history = module.terminal_history();
11678            if history.entries.len() == 2 {
11679                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11680                assert_eq!(history.dropped, 0);
11681                assert_eq!(
11682                    history
11683                        .entries
11684                        .iter()
11685                        .map(|entry| entry.exit_code)
11686                        .collect::<Vec<_>>(),
11687                    vec![Some(23), Some(23)]
11688                );
11689                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11690                return;
11691            }
11692            assert!(
11693                Instant::now() < deadline,
11694                "module did not retain two terminal exits: {history:?}"
11695            );
11696            sleep(Duration::from_millis(10)).await;
11697        }
11698    }
11699
11700    /// A disable issued while a crash respawn is still backing off must preempt
11701    /// that respawn: the operator's stop wins, the disable must not queue behind
11702    /// the backoff, and the module must never come back up afterwards.
11703    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11704    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11705        let backoff = Duration::from_secs(2);
11706        let supervisor = Supervisor::new_for_test(
11707            Arc::new(Registry::default()),
11708            RestartPolicy::new(10, backoff),
11709        );
11710        let module = supervisor
11711            .spawn(ModuleSpec {
11712                module_id: "disable-during-backoff".to_string(),
11713                program: fake_aft_stub_path(),
11714                args: Vec::new(),
11715                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11716                reserved: false,
11717                reserved_prefixes: Vec::new(),
11718                protocol: ModuleProtocol::Subc,
11719                overlap: Default::default(),
11720            })
11721            .unwrap();
11722
11723        // Wait for the first crash to put the module into its backoff window.
11724        let deadline = Instant::now() + Duration::from_secs(5);
11725        loop {
11726            if module.status().unwrap().state == ModuleState::Restarting {
11727                break;
11728            }
11729            assert!(
11730                Instant::now() < deadline,
11731                "module never entered the crash backoff"
11732            );
11733            sleep(Duration::from_millis(10)).await;
11734        }
11735
11736        let started = Instant::now();
11737        module.set_enabled(false).await.unwrap();
11738        let waited = started.elapsed();
11739
11740        assert!(
11741            waited < backoff / 2,
11742            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11743        );
11744        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11745
11746        // Outlast the backoff: the respawn it was counting down to must never run.
11747        sleep(backoff + Duration::from_millis(500)).await;
11748        let status = module.status().unwrap();
11749        assert_eq!(status.state, ModuleState::Disabled);
11750        assert_eq!(
11751            status.spawn_generation, 1,
11752            "module respawned after the operator disabled it"
11753        );
11754    }
11755
11756    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11757    /// the shape of nats-server, the program this rule exists for.
11758    #[cfg(unix)]
11759    fn protocol_none_sigterm_exits_clean_spec(
11760        module_id: &str,
11761        dir: &std::path::Path,
11762    ) -> (ModuleSpec, PathBuf, PathBuf) {
11763        let ready = dir.join("ready");
11764        let marker = dir.join("sigterm");
11765        let spec = ModuleSpec {
11766            module_id: module_id.to_string(),
11767            program: fake_aft_stub_path(),
11768            args: Vec::new(),
11769            env: vec![
11770                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11771                (
11772                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11773                    marker.display().to_string(),
11774                ),
11775                (
11776                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11777                    ready.display().to_string(),
11778                ),
11779            ],
11780            reserved: false,
11781            reserved_prefixes: Vec::new(),
11782            protocol: ModuleProtocol::None,
11783            overlap: Default::default(),
11784        };
11785        (spec, ready, marker)
11786    }
11787
11788    /// Wait for a file the child writes, so a signal is never sent before the
11789    /// child's SIGTERM handler is installed (the default disposition would
11790    /// kill it by signal and the exit would not be clean).
11791    #[cfg(unix)]
11792    async fn wait_for_file(path: &std::path::Path) {
11793        let deadline = Instant::now() + Duration::from_secs(10);
11794        while !path.exists() {
11795            assert!(
11796                Instant::now() < deadline,
11797                "{} never appeared",
11798                path.display()
11799            );
11800            sleep(Duration::from_millis(10)).await;
11801        }
11802    }
11803
11804    /// A protocol-none module that exits 0 because something OUTSIDE the
11805    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11806    /// the crash-path disposition rather than `stopped`.
11807    #[cfg(unix)]
11808    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11809    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11810        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11811        let (spec, ready, marker) =
11812            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11813        let supervisor = Supervisor::new_for_test(
11814            Arc::new(Registry::default()),
11815            RestartPolicy::new(3, Duration::ZERO),
11816        );
11817        let module = supervisor.spawn(spec).unwrap();
11818        wait_for_file(&ready).await;
11819        // The ready file proves the child installed its SIGTERM handler, not
11820        // that the supervisor has processed the privacy trampoline's exec
11821        // acknowledgement. On macOS status withholds the pid until then.
11822        let deadline = Instant::now() + Duration::from_secs(10);
11823        let first_pid = loop {
11824            let status = module.status().unwrap();
11825            if status.state == ModuleState::Running {
11826                if let Some(pid) = status.pid {
11827                    break pid;
11828                }
11829            }
11830            assert!(
11831                Instant::now() < deadline,
11832                "a running module must report its pid after exec confirmation: {status:?}"
11833            );
11834            sleep(Duration::from_millis(10)).await;
11835        };
11836
11837        rustix::process::kill_process(
11838            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11839            rustix::process::Signal::TERM,
11840        )
11841        .unwrap();
11842
11843        let deadline = Instant::now() + Duration::from_secs(10);
11844        let respawned = loop {
11845            let status = module.status().unwrap();
11846            if status.state == ModuleState::Running
11847                && status.pid.is_some_and(|pid| pid != first_pid)
11848            {
11849                break status;
11850            }
11851            assert!(
11852                Instant::now() < deadline,
11853                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11854            );
11855            sleep(Duration::from_millis(10)).await;
11856        };
11857        assert_eq!(respawned.spawn_generation, 2);
11858        assert!(
11859            marker.exists(),
11860            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11861        );
11862
11863        let history = module.terminal_history();
11864        assert_eq!(history.entries.len(), 1, "{history:?}");
11865        let entry = &history.entries[0];
11866        assert_eq!(entry.exit_code, Some(0));
11867        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11868        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11869
11870        module.stop().await.unwrap();
11871    }
11872
11873    /// Repeated unrequested clean exits of a protocol-none module spend the
11874    /// restart budget exactly as crashes do, and the module ends `failed` with
11875    /// the budget named.
11876    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11877    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11878        let supervisor = Supervisor::new_for_test(
11879            Arc::new(Registry::default()),
11880            RestartPolicy::new(1, Duration::ZERO),
11881        );
11882        let module = supervisor
11883            .spawn(ModuleSpec {
11884                module_id: "none-clean-exit-budget".to_string(),
11885                program: fake_aft_stub_path(),
11886                args: Vec::new(),
11887                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11888                reserved: false,
11889                reserved_prefixes: Vec::new(),
11890                protocol: ModuleProtocol::None,
11891                overlap: Default::default(),
11892            })
11893            .unwrap();
11894
11895        // Failed follows the terminal write; this deadline only bounds a hang,
11896        // not an assumed duration for the two launches or their exit recording.
11897        let deadline = Instant::now() + Duration::from_secs(10);
11898        loop {
11899            let status = module.status().unwrap();
11900            if status.state == ModuleState::Failed {
11901                break;
11902            }
11903            assert!(
11904                Instant::now() < deadline,
11905                "module never exhausted its budget: {status:?} {:?}",
11906                module.terminal_history()
11907            );
11908            sleep(Duration::from_millis(10)).await;
11909        }
11910        let history = module.terminal_history();
11911        assert_eq!(
11912            history
11913                .entries
11914                .iter()
11915                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11916                .collect::<Vec<_>>(),
11917            vec![
11918                (Some(0), TerminalDisposition::Restarting),
11919                (Some(0), TerminalDisposition::Failed),
11920            ]
11921        );
11922        let detail = history.entries[1]
11923            .disposition_detail
11924            .as_deref()
11925            .expect("a budget failure names the budget");
11926        assert!(detail.contains("max_restarts=1"), "{detail}");
11927        assert_eq!(module.status().unwrap().spawn_generation, 2);
11928    }
11929
11930    #[test]
11931    fn restart_budget_failure_is_published_after_its_terminal_record() {
11932        use std::io::{Read, Write};
11933        let supervisor = Supervisor::new_for_test(
11934            Arc::new(Registry::default()),
11935            RestartPolicy::new(0, Duration::ZERO),
11936        );
11937        let runtime = supervisor.runtime_config();
11938        let spec = ModuleSpec {
11939            protocol: ModuleProtocol::None,
11940            ..windowed_crash_spec("budget-publication-order")
11941        };
11942        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11943            ModuleState::Running,
11944            true,
11945        )));
11946        lock_snapshot(&snapshot).unwrap().process_alive = true;
11947        let reaped_snapshot = snapshot.clone();
11948        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11949        let history = runtime.terminal_ring.clone();
11950        let reap = std::thread::spawn(move || {
11951            tokio::runtime::Builder::new_current_thread()
11952                .enable_all()
11953                .build()
11954                .unwrap()
11955                .block_on(on_child_exit(
11956                    &spec,
11957                    runtime.restart_policy,
11958                    &supervisor.registry,
11959                    &reaped_snapshot,
11960                    &runtime.terminal_ring,
11961                    &runtime.spawn_events,
11962                    &runtime.child_roster,
11963                    ExitReport {
11964                        kind: ExitKind::Clean,
11965                        code: Some(0),
11966                        signal: None,
11967                        at_ms: 1,
11968                    },
11969                ))
11970        });
11971        let mut reached = [0];
11972        barrier.read_exact(&mut reached).unwrap();
11973        assert_eq!(&reached, b"R");
11974        let (before_state, before_alive) = {
11975            let state = lock_snapshot(&snapshot).unwrap();
11976            (state.state, state.process_alive)
11977        };
11978        let before_history = history.lock().unwrap().snapshot();
11979        barrier.write_all(b"X").unwrap();
11980        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11981        assert!(before_history.entries.is_empty());
11982        assert_ne!(
11983            before_state,
11984            ModuleState::Failed,
11985            "Failed was visible before its terminal record could be written"
11986        );
11987        assert!(before_alive);
11988        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11989        let history = history.lock().unwrap().snapshot();
11990        assert_eq!(history.entries.len(), 1);
11991        assert_eq!(history.entries[0].exit_code, Some(0));
11992        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11993    }
11994
11995    #[test]
11996    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11997        use std::io::{Read, Write};
11998
11999        let supervisor = Supervisor::new_for_test(
12000            Arc::new(Registry::default()),
12001            RestartPolicy::new(3, Duration::ZERO),
12002        );
12003        let mut runtime = supervisor.runtime_config();
12004        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12005        let snapshot = module.inner.snapshot.clone();
12006        lock_snapshot(&snapshot).unwrap().process_alive = true;
12007        runtime.terminal_ring = module.inner.terminal_ring.clone();
12008        let spec = windowed_crash_spec(module.module_id());
12009        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12010        let reaped_snapshot = snapshot.clone();
12011        let reap = std::thread::spawn(move || {
12012            tokio::runtime::Builder::new_current_thread()
12013                .enable_all()
12014                .build()
12015                .unwrap()
12016                .block_on(on_child_exit(
12017                    &spec,
12018                    runtime.restart_policy,
12019                    &supervisor.registry,
12020                    &reaped_snapshot,
12021                    &runtime.terminal_ring,
12022                    &runtime.spawn_events,
12023                    &runtime.child_roster,
12024                    ExitReport {
12025                        kind: ExitKind::Crash,
12026                        code: Some(2),
12027                        signal: None,
12028                        at_ms: 1,
12029                    },
12030                ))
12031        });
12032
12033        // Hold the crash handler between its budget decision and publication.
12034        // Connection cleanup must still see the recovery that this crash chose,
12035        // even when that recovery consumes the last available restart slot.
12036        let mut reached = [0];
12037        barrier.read_exact(&mut reached).unwrap();
12038        assert_eq!(&reached, b"R");
12039        let before_status = module.status().unwrap();
12040        let before_history = module.terminal_history();
12041        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12042        barrier.write_all(b"X").unwrap();
12043        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12044
12045        assert_eq!(before_status.state, ModuleState::Running);
12046        assert!(before_status.process_alive);
12047        assert_eq!(before_status.restart_count, 3);
12048        assert!(before_history.entries.is_empty());
12049        assert!(
12050            before_recovers,
12051            "connection cleanup would report a terminal close for a scheduled crash restart"
12052        );
12053        let restarting = module.status().unwrap();
12054        assert_eq!(restarting.state, ModuleState::Restarting);
12055        assert_eq!(restarting.restart_count, 3);
12056        assert!(module.will_recover_after_connection_loss().unwrap());
12057        let history = module.terminal_history();
12058        assert_eq!(history.entries.len(), 1);
12059        assert_eq!(
12060            history.entries[0].disposition,
12061            TerminalDisposition::Restarting
12062        );
12063
12064        // Once the replacement is running, its next crash must be judged
12065        // against the exhausted budget rather than the previous recovery.
12066        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12067            state.state = ModuleState::Running;
12068            state.respawn_pending = false;
12069        })
12070        .unwrap();
12071        assert!(!module.will_recover_after_connection_loss().unwrap());
12072        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12073            state.enabled = false;
12074            state.respawn_pending = true;
12075        })
12076        .unwrap();
12077        assert!(!module.will_recover_after_connection_loss().unwrap());
12078    }
12079
12080    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12081        use std::net::{TcpListener, TcpStream};
12082        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12083        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12084        let (barrier, _) = listener.accept().unwrap();
12085        for stream in [&writer, &barrier] {
12086            stream
12087                .set_read_timeout(Some(Duration::from_secs(10)))
12088                .unwrap();
12089            stream
12090                .set_write_timeout(Some(Duration::from_secs(10)))
12091                .unwrap();
12092        }
12093        *events.1.lock().unwrap() = Some(writer);
12094        barrier
12095    }
12096
12097    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12098        use std::io::{Read, Write};
12099        let barrier = events.1.lock().unwrap().take();
12100        if let Some(mut barrier) = barrier {
12101            barrier.write_all(b"R").unwrap();
12102            let mut release = [0];
12103            barrier.read_exact(&mut release).unwrap();
12104            assert_eq!(&release, b"X");
12105        }
12106    }
12107
12108    #[test]
12109    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12110        use std::io::{Read, Write};
12111
12112        let supervisor = Supervisor::new_for_test(
12113            Arc::new(Registry::default()),
12114            RestartPolicy::new(0, Duration::ZERO),
12115        );
12116        let mut runtime = supervisor.runtime_config();
12117        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12118        let snapshot = module.inner.snapshot.clone();
12119        runtime.terminal_ring = module.inner.terminal_ring.clone();
12120        let spec = ModuleSpec {
12121            module_id: module.module_id().into(),
12122            program: fake_aft_stub_path(),
12123            protocol: ModuleProtocol::None,
12124            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12125            ..windowed_crash_spec(module.module_id())
12126        };
12127        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12128        let reap = std::thread::spawn(move || {
12129            tokio::runtime::Builder::new_current_thread()
12130                .enable_all()
12131                .build()
12132                .unwrap()
12133                .block_on(async {
12134                    let mut child =
12135                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12136                    health_restart_child(
12137                        &spec,
12138                        &runtime,
12139                        &supervisor.registry,
12140                        &supervisor.process_liveness,
12141                        &snapshot,
12142                        &mut child,
12143                        SupervisorHealthStatus::Failing,
12144                        Some("health probe did not answer"),
12145                        1,
12146                    )
12147                    .await
12148                    .unwrap();
12149                    assert!(child.is_none());
12150                });
12151        });
12152        // The loopback barrier acknowledges entry into terminal recording by
12153        // health_restart_child's drain, then waits for the test's release byte.
12154        // Status and history are observed while that writer is held, not after
12155        // an elapsed-time guess about when the health action has completed.
12156        let mut reached = [0];
12157        barrier.read_exact(&mut reached).unwrap();
12158        assert_eq!(&reached, b"R");
12159        let before_status = module.status().unwrap();
12160        let before_history = module.terminal_history();
12161        barrier.write_all(b"X").unwrap();
12162        reap.join().unwrap();
12163        assert!(before_history.entries.is_empty());
12164        assert_ne!(
12165            before_status.state,
12166            ModuleState::Failed,
12167            "Failed was visible before its terminal record could be written"
12168        );
12169        assert!(
12170            before_status.process_alive,
12171            "process death was visible before its terminal record could be written"
12172        );
12173        let failed = module.status().unwrap();
12174        assert_eq!(failed.state, ModuleState::Failed);
12175        assert!(!failed.process_alive);
12176        let history = module.terminal_history();
12177        assert_eq!(history.entries.len(), 1);
12178        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12179        assert_eq!(
12180            history.entries[0].disposition_detail.as_deref(),
12181            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12182        );
12183    }
12184
12185    #[test]
12186    fn observed_exits_publish_process_death_after_terminal_history() {
12187        use std::io::{Read, Write};
12188        for (kind, enabled, shutdown, wait_error, expected_state) in [
12189            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12190            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12191            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12192            (
12193                ExitKind::DeliberateSeverance,
12194                true,
12195                false,
12196                false,
12197                ModuleState::Restarting,
12198            ),
12199            #[cfg(unix)]
12200            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12201            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12202        ] {
12203            let supervisor = Supervisor::new_for_test(
12204                Arc::new(Registry::default()),
12205                RestartPolicy::new(1, Duration::ZERO),
12206            );
12207            let runtime = supervisor.runtime_config();
12208            #[cfg(unix)]
12209            if shutdown {
12210                runtime.child_roster.close();
12211            }
12212            #[cfg(not(unix))]
12213            let _ = shutdown;
12214            let spec = windowed_crash_spec("exit-publication-order");
12215            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12216                ModuleState::Running,
12217                enabled,
12218            )));
12219            lock_snapshot(&snapshot).unwrap().process_alive = true;
12220            let reaped_snapshot = snapshot.clone();
12221            let history = runtime.terminal_ring.clone();
12222            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12223            let reap = std::thread::spawn(move || {
12224                if wait_error {
12225                    super::fail_after_wait_error(
12226                        &spec.module_id,
12227                        &reaped_snapshot,
12228                        &runtime.terminal_ring,
12229                        &runtime.spawn_events,
12230                        "wait failed".into(),
12231                    );
12232                } else {
12233                    tokio::runtime::Builder::new_current_thread()
12234                        .enable_all()
12235                        .build()
12236                        .unwrap()
12237                        .block_on(on_child_exit(
12238                            &spec,
12239                            runtime.restart_policy,
12240                            &supervisor.registry,
12241                            &reaped_snapshot,
12242                            &runtime.terminal_ring,
12243                            &runtime.spawn_events,
12244                            &runtime.child_roster,
12245                            ExitReport {
12246                                kind,
12247                                code: Some(0),
12248                                signal: None,
12249                                at_ms: 1,
12250                            },
12251                        ));
12252                }
12253            });
12254            let mut reached = [0];
12255            barrier.read_exact(&mut reached).unwrap();
12256            assert_eq!(&reached, b"R");
12257            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12258            let before_history = history.lock().unwrap().snapshot();
12259            barrier.write_all(b"X").unwrap();
12260            reap.join().unwrap();
12261            assert!(
12262                before_alive,
12263                "{kind:?}: death was published before recording"
12264            );
12265            assert!(before_history.entries.is_empty());
12266            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12267            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12268            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12269        }
12270    }
12271
12272    #[tokio::test]
12273    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12274        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12275        let path = dir.join("terminals.jsonl");
12276        std::fs::create_dir(&path).unwrap();
12277        let supervisor = Supervisor::new_for_test(
12278            Arc::new(Registry::default()),
12279            RestartPolicy::new(0, Duration::ZERO),
12280        )
12281        .with_terminal_journal(path, "health-budget-journal-failure".into());
12282        let runtime = supervisor.runtime_config();
12283        let spec = ModuleSpec {
12284            program: fake_aft_stub_path(),
12285            protocol: ModuleProtocol::None,
12286            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12287            ..windowed_crash_spec("health-budget-journal-failure")
12288        };
12289        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12290        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12291        health_restart_child(
12292            &spec,
12293            &runtime,
12294            &supervisor.registry,
12295            &supervisor.process_liveness,
12296            &snapshot,
12297            &mut child,
12298            SupervisorHealthStatus::Failing,
12299            None,
12300            1,
12301        )
12302        .await
12303        .unwrap();
12304        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12305        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12306        let history = runtime
12307            .terminal_ring
12308            .lock()
12309            .unwrap()
12310            .durable_history(&spec.module_id);
12311        assert!(history.journal_write_failures > 0);
12312        assert_eq!(history.entries.len(), 1);
12313        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12314    }
12315
12316    #[test]
12317    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12318        let supervisor = Supervisor::new_for_test(
12319            Arc::new(Registry::default()),
12320            RestartPolicy::new(0, Duration::ZERO),
12321        );
12322        let runtime = supervisor.runtime_config();
12323        let spec = ModuleSpec {
12324            program: fake_aft_stub_path(),
12325            protocol: ModuleProtocol::None,
12326            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12327            ..windowed_crash_spec("health-budget-recording-panic")
12328        };
12329        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12330        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12331            tokio::runtime::Builder::new_current_thread()
12332                .enable_all()
12333                .build()
12334                .unwrap()
12335                .block_on(async {
12336                    let mut child =
12337                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12338                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12339                    health_restart_child(
12340                        &spec,
12341                        &runtime,
12342                        &supervisor.registry,
12343                        &supervisor.process_liveness,
12344                        &snapshot,
12345                        &mut child,
12346                        SupervisorHealthStatus::Failing,
12347                        None,
12348                        1,
12349                    )
12350                    .await
12351                    .unwrap();
12352                });
12353        }));
12354        let panic = result.expect_err("recording must still unwind");
12355        assert_eq!(
12356            panic.downcast_ref::<String>().map(String::as_str),
12357            Some("spawn event sequence exhausted")
12358        );
12359        let state = lock_snapshot(&snapshot).unwrap();
12360        assert_eq!(state.state, ModuleState::Failed);
12361        assert!(!state.process_alive);
12362        assert!(state.last_exit.is_some());
12363    }
12364
12365    #[tokio::test]
12366    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12367        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12368        let path = dir.join("terminals.jsonl");
12369        std::fs::create_dir(&path).unwrap();
12370        let supervisor = Supervisor::new_for_test(
12371            Arc::new(Registry::default()),
12372            RestartPolicy::new(0, Duration::ZERO),
12373        )
12374        .with_terminal_journal(path, "budget-journal-failure".into());
12375        let runtime = supervisor.runtime_config();
12376        let spec = windowed_crash_spec("budget-journal-failure");
12377        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12378        assert!(matches!(
12379            on_child_exit(
12380                &spec,
12381                runtime.restart_policy,
12382                &supervisor.registry,
12383                &snapshot,
12384                &runtime.terminal_ring,
12385                &runtime.spawn_events,
12386                &runtime.child_roster,
12387                crash_exit_report(1),
12388            )
12389            .await,
12390            NextAction::Stop { .. }
12391        ));
12392        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12393        let history = runtime
12394            .terminal_ring
12395            .lock()
12396            .unwrap()
12397            .durable_history(&spec.module_id);
12398        assert!(history.journal_write_failures > 0);
12399        assert_eq!(history.entries.len(), 1);
12400        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12401    }
12402
12403    #[test]
12404    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12405        let supervisor = Supervisor::new_for_test(
12406            Arc::new(Registry::default()),
12407            RestartPolicy::new(0, Duration::ZERO),
12408        );
12409        let runtime = supervisor.runtime_config();
12410        let spec = windowed_crash_spec("budget-recording-panic");
12411        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12412        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12413        // Exhausting the event sequence makes emit_exited panic before the
12414        // terminal write, exercising publication on the recording unwind.
12415        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12416        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12417            tokio::runtime::Builder::new_current_thread()
12418                .enable_all()
12419                .build()
12420                .unwrap()
12421                .block_on(on_child_exit(
12422                    &spec,
12423                    runtime.restart_policy,
12424                    &supervisor.registry,
12425                    &snapshot,
12426                    &runtime.terminal_ring,
12427                    &runtime.spawn_events,
12428                    &runtime.child_roster,
12429                    crash_exit_report(1),
12430                ))
12431        }));
12432        let panic = result.err().expect("recording must still unwind");
12433        assert_eq!(
12434            panic.downcast_ref::<String>().map(String::as_str),
12435            Some("spawn event sequence exhausted")
12436        );
12437        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12438    }
12439
12440    /// A stop the supervisor itself requests still stops a protocol-none
12441    /// module, even though the child answers the SIGTERM with exit 0.
12442    #[cfg(unix)]
12443    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12444    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12445        for disable in [false, true] {
12446            let label = if disable {
12447                "none-requested-disable"
12448            } else {
12449                "none-requested-stop"
12450            };
12451            let dir = cortexkit_test_support::ScratchDir::new(label);
12452            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12453            let supervisor = Supervisor::new_for_test(
12454                Arc::new(Registry::default()),
12455                RestartPolicy::new(3, Duration::ZERO),
12456            );
12457            let module = supervisor.spawn(spec).unwrap();
12458            wait_for_file(&ready).await;
12459
12460            if disable {
12461                module.set_enabled(false).await.unwrap();
12462            } else {
12463                module.stop().await.unwrap();
12464            }
12465            assert!(
12466                marker.exists(),
12467                "{label}: the child must have left through its SIGTERM handler with exit 0"
12468            );
12469
12470            // Long enough for a zero-backoff respawn to have happened if the
12471            // exit had been treated as a crash.
12472            sleep(Duration::from_millis(500)).await;
12473            let status = module.status().unwrap();
12474            let expected = if disable {
12475                ModuleState::Disabled
12476            } else {
12477                ModuleState::Stopped
12478            };
12479            assert_eq!(status.state, expected, "{label}");
12480            assert_eq!(
12481                status.spawn_generation, 1,
12482                "{label}: respawned after a requested stop"
12483            );
12484            let history = module.terminal_history();
12485            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12486            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12487            assert_ne!(
12488                history.entries[0].disposition,
12489                TerminalDisposition::Restarting,
12490                "{label}"
12491            );
12492        }
12493    }
12494
12495    /// A subc-wire module that exits 0 on its own is still a stop: the
12496    /// protocol-none rule must not reach it.
12497    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12498    async fn subc_wire_clean_exit_is_still_a_stop() {
12499        let supervisor = Supervisor::new_for_test(
12500            Arc::new(Registry::default()),
12501            RestartPolicy::new(3, Duration::ZERO),
12502        );
12503        let module = supervisor
12504            .spawn(ModuleSpec {
12505                module_id: "wire-clean-exit".to_string(),
12506                program: fake_aft_stub_path(),
12507                args: Vec::new(),
12508                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12509                reserved: false,
12510                reserved_prefixes: Vec::new(),
12511                protocol: ModuleProtocol::Subc,
12512                overlap: Default::default(),
12513            })
12514            .unwrap();
12515
12516        let deadline = Instant::now() + Duration::from_secs(10);
12517        while module.terminal_history().entries.is_empty() {
12518            assert!(Instant::now() < deadline, "module never exited");
12519            sleep(Duration::from_millis(10)).await;
12520        }
12521        // Long enough for a zero-backoff respawn to have happened.
12522        sleep(Duration::from_millis(500)).await;
12523        let status = module.status().unwrap();
12524        assert_eq!(status.state, ModuleState::Stopped);
12525        assert_eq!(status.spawn_generation, 1);
12526        let history = module.terminal_history();
12527        assert_eq!(history.entries.len(), 1, "{history:?}");
12528        assert_eq!(history.entries[0].exit_code, Some(0));
12529        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12530    }
12531
12532    #[cfg(unix)]
12533    #[tokio::test]
12534    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12535        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12536        let record = dir.join("live-children.json");
12537        let supervisor = Supervisor::new_for_test(
12538            Arc::new(Registry::default()),
12539            RestartPolicy::new(0, Duration::ZERO),
12540        );
12541        let mut runtime = supervisor.runtime_config();
12542        runtime.child_roster.record_to(record.clone());
12543        let gate = Arc::new(super::ReloadExitRecordGate::default());
12544        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12545        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12546        let spec = ModuleSpec {
12547            module_id: "reload-exit-roster".into(),
12548            program: fake_aft_stub_path(),
12549            args: Vec::new(),
12550            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12551            reserved: false,
12552            reserved_prefixes: Vec::new(),
12553            protocol: ModuleProtocol::Subc,
12554            overlap: Default::default(),
12555        };
12556        let mut child = None;
12557        let reload = super::finish_reload_child(
12558            &spec,
12559            &runtime,
12560            &supervisor.registry,
12561            &supervisor.process_liveness,
12562            &snapshot,
12563            &mut child,
12564        );
12565        tokio::pin!(reload);
12566        tokio::select! {
12567            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12568            _ = gate.reached.notified() => {}
12569        }
12570        assert!(runtime
12571            .terminal_ring
12572            .lock()
12573            .unwrap()
12574            .snapshot()
12575            .entries
12576            .is_empty());
12577        assert_eq!(
12578            crate::live_children::read_record(&record).unwrap().len(),
12579            1,
12580            "shutdown must still wait for the reaped child until its terminal record exists"
12581        );
12582        runtime.child_roster.close();
12583        gate.resume.notify_one();
12584        assert!(reload.await.is_err());
12585        assert!(crate::live_children::read_record(&record)
12586            .unwrap()
12587            .is_empty());
12588        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12589        assert_eq!(history.entries.len(), 1);
12590        assert_eq!(
12591            history.entries[0].disposition,
12592            TerminalDisposition::DaemonShutdown
12593        );
12594    }
12595
12596    /// Each restart-producing arm has its own state transition. Keeping their
12597    /// lifetime count assertions adjacent prevents a later new arm from silently
12598    /// spending budget without recording the historical restart.
12599    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12600    async fn every_restart_increment_path_advances_lifetime_count() {
12601        let supervisor = Supervisor::new_for_test(
12602            Arc::new(Registry::default()),
12603            RestartPolicy::new(1, Duration::ZERO),
12604        );
12605        let runtime = supervisor.runtime_config();
12606        let spec = ModuleSpec {
12607            module_id: "lifetime-increment-path".to_string(),
12608            program: PathBuf::from("/unused/lifetime-increment-path"),
12609            args: Vec::new(),
12610            env: Vec::new(),
12611            reserved: false,
12612            reserved_prefixes: Vec::new(),
12613            protocol: ModuleProtocol::Subc,
12614            overlap: Default::default(),
12615        };
12616
12617        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12618        assert!(matches!(
12619            on_child_exit(
12620                &spec,
12621                runtime.restart_policy,
12622                &supervisor.registry,
12623                &crash_snapshot,
12624                &runtime.terminal_ring,
12625                &runtime.spawn_events,
12626                &runtime.child_roster,
12627                ExitReport {
12628                    kind: ExitKind::Crash,
12629                    code: Some(1),
12630                    signal: None,
12631                    at_ms: 1,
12632                },
12633            )
12634            .await,
12635            NextAction::Restart { schedule: _ }
12636        ));
12637        let (crash_restarts, crash_lifetime) = {
12638            let state = lock_snapshot(&crash_snapshot).unwrap();
12639            (state.crash_restarts.len(), state.lifetime_restarts)
12640        };
12641        assert_eq!(crash_restarts, 1);
12642        assert_eq!(crash_lifetime, 1);
12643
12644        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12645        let mut health_child = None;
12646        assert!(matches!(
12647            health_restart_child(
12648                &spec,
12649                &runtime,
12650                &supervisor.registry,
12651                &supervisor.process_liveness,
12652                &health_snapshot,
12653                &mut health_child,
12654                SupervisorHealthStatus::Failing,
12655                None,
12656                2,
12657            )
12658            .await,
12659            Ok(())
12660        ));
12661        assert!(health_child.is_none());
12662        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12663        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12664        let (health_restarts, health_lifetime) = {
12665            let state = lock_snapshot(&health_snapshot).unwrap();
12666            (state.crash_restarts.len(), state.lifetime_restarts)
12667        };
12668        assert_eq!(health_restarts, 1);
12669        assert_eq!(health_lifetime, 1);
12670
12671        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12672        let mut reload_child = None;
12673        assert!(matches!(
12674            handle_reload_spawn_failure(
12675                &spec,
12676                &runtime,
12677                &supervisor.process_liveness,
12678                &reload_snapshot,
12679                &mut reload_child,
12680                "forced reload spawn failure".to_string(),
12681            )
12682            .await,
12683            Err(SuperviseError::ReloadFailed { .. })
12684        ));
12685        let (reload_restarts, reload_lifetime) = {
12686            let state = lock_snapshot(&reload_snapshot).unwrap();
12687            (state.crash_restarts.len(), state.lifetime_restarts)
12688        };
12689        assert_eq!(reload_restarts, 1);
12690        assert_eq!(reload_lifetime, 1);
12691    }
12692
12693    #[tokio::test]
12694    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12695        let supervisor = Supervisor::new_for_test(
12696            Arc::new(Registry::default()),
12697            RestartPolicy::new(3, Duration::ZERO),
12698        );
12699        let runtime = supervisor.runtime_config();
12700        let spec = ModuleSpec {
12701            module_id: "deliberately-severed".to_string(),
12702            program: PathBuf::from("/unused/deliberately-severed"),
12703            args: Vec::new(),
12704            env: Vec::new(),
12705            reserved: false,
12706            reserved_prefixes: Vec::new(),
12707            protocol: ModuleProtocol::Subc,
12708            overlap: Default::default(),
12709        };
12710        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12711        let process = ProcessIdentity {
12712            pid: 41,
12713            start_time: 101,
12714        };
12715        record_deliberate_severance(&snapshot, process).unwrap();
12716        let exit_report = apply_deliberate_severance_marker(
12717            &snapshot,
12718            Some(process),
12719            ExitReport {
12720                kind: ExitKind::Crash,
12721                code: Some(1),
12722                signal: None,
12723                at_ms: 1,
12724            },
12725        );
12726        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12727
12728        assert!(matches!(
12729            on_child_exit(
12730                &spec,
12731                runtime.restart_policy,
12732                &supervisor.registry,
12733                &snapshot,
12734                &runtime.terminal_ring,
12735                &runtime.spawn_events,
12736                &runtime.child_roster,
12737                exit_report,
12738            )
12739            .await,
12740            NextAction::Restart { schedule: _ }
12741        ));
12742        let state = lock_snapshot(&snapshot).unwrap();
12743        assert_eq!(state.lifetime_restarts, 1);
12744        assert_eq!(state.crash_restarts.len(), 0);
12745    }
12746
12747    #[tokio::test]
12748    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12749        let supervisor = Supervisor::new_for_test(
12750            Arc::new(Registry::default()),
12751            RestartPolicy::new(3, Duration::ZERO),
12752        );
12753        let runtime = supervisor.runtime_config();
12754        let spec = ModuleSpec {
12755            module_id: "genuine-crash".to_string(),
12756            program: PathBuf::from("/unused/genuine-crash"),
12757            args: Vec::new(),
12758            env: Vec::new(),
12759            reserved: false,
12760            reserved_prefixes: Vec::new(),
12761            protocol: ModuleProtocol::Subc,
12762            overlap: Default::default(),
12763        };
12764        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12765
12766        assert!(matches!(
12767            on_child_exit(
12768                &spec,
12769                runtime.restart_policy,
12770                &supervisor.registry,
12771                &snapshot,
12772                &runtime.terminal_ring,
12773                &runtime.spawn_events,
12774                &runtime.child_roster,
12775                ExitReport {
12776                    kind: ExitKind::Crash,
12777                    code: Some(1),
12778                    signal: None,
12779                    at_ms: 1,
12780                },
12781            )
12782            .await,
12783            NextAction::Restart { schedule: _ }
12784        ));
12785        let state = lock_snapshot(&snapshot).unwrap();
12786        assert_eq!(state.lifetime_restarts, 1);
12787        assert_eq!(state.crash_restarts.len(), 1);
12788    }
12789
12790    fn crash_exit_report(at_ms: u64) -> ExitReport {
12791        ExitReport {
12792            kind: ExitKind::Crash,
12793            code: Some(1),
12794            signal: None,
12795            at_ms,
12796        }
12797    }
12798
12799    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12800        ModuleSpec {
12801            module_id: module_id.to_string(),
12802            program: PathBuf::from("/unused").join(module_id),
12803            args: Vec::new(),
12804            env: Vec::new(),
12805            reserved: false,
12806            reserved_prefixes: Vec::new(),
12807            protocol: ModuleProtocol::Subc,
12808            overlap: Default::default(),
12809        }
12810    }
12811
12812    /// A real crash loop still stops. Three crashes with nothing aging out spend
12813    /// a budget of two and the third respawn is refused, and both surfaces an
12814    /// operator has -- the log line and the retained terminal record -- name the
12815    /// window rather than only the cap, because `max_restarts=2` alone is what
12816    /// this budget used to mean.
12817    #[tokio::test]
12818    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12819        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12820        let supervisor = Supervisor::new_for_test(
12821            Arc::new(Registry::default()),
12822            RestartPolicy::new(2, Duration::ZERO),
12823        );
12824        let runtime = supervisor.runtime_config();
12825        let spec = windowed_crash_spec("crash-loop-in-window");
12826        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12827
12828        for attempt in 1..=2 {
12829            assert!(
12830                matches!(
12831                    on_child_exit(
12832                        &spec,
12833                        runtime.restart_policy,
12834                        &supervisor.registry,
12835                        &snapshot,
12836                        &runtime.terminal_ring,
12837                        &runtime.spawn_events,
12838                        &runtime.child_roster,
12839                        crash_exit_report(attempt),
12840                    )
12841                    .await,
12842                    NextAction::Restart { schedule: _ }
12843                ),
12844                "crash {attempt} is inside the budget and must respawn"
12845            );
12846        }
12847
12848        assert!(matches!(
12849            on_child_exit(
12850                &spec,
12851                runtime.restart_policy,
12852                &supervisor.registry,
12853                &snapshot,
12854                &runtime.terminal_ring,
12855                &runtime.spawn_events,
12856                &runtime.child_roster,
12857                crash_exit_report(3),
12858            )
12859            .await,
12860            NextAction::Stop { .. }
12861        ));
12862
12863        {
12864            let state = lock_snapshot(&snapshot).unwrap();
12865            assert_eq!(state.state, ModuleState::Failed);
12866            assert_eq!(state.crash_restarts.len(), 2);
12867            assert_eq!(state.lifetime_restarts, 2);
12868        }
12869
12870        let history = runtime
12871            .terminal_ring
12872            .lock()
12873            .expect("terminal ring is not poisoned")
12874            .snapshot();
12875        let last = history
12876            .entries
12877            .last()
12878            .expect("the refused crash is retained");
12879        assert_eq!(last.disposition, TerminalDisposition::Failed);
12880        assert_eq!(
12881            last.disposition_detail.as_deref(),
12882            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12883        );
12884
12885        let captured = crate::router::test_log::captured_logs(&logs);
12886        assert!(
12887            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12888            "the stop must be logged with its window: {captured}"
12889        );
12890    }
12891
12892    /// The rate, stated as a test: three crashes where the first has aged past
12893    /// the window are two crashes as far as the budget is concerned, so the
12894    /// third respawn is allowed and the ring holds only the two recent ones.
12895    ///
12896    /// This is the case a lifetime counter got wrong -- and the case the daemon
12897    /// now hits routinely, since a module exits non-zero every time its
12898    /// connection to the daemon drops.
12899    #[tokio::test]
12900    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12901        let supervisor = Supervisor::new_for_test(
12902            Arc::new(Registry::default()),
12903            RestartPolicy::new(2, Duration::ZERO),
12904        );
12905        let runtime = supervisor.runtime_config();
12906        let spec = windowed_crash_spec("crash-across-windows");
12907        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12908
12909        for attempt in 1..=2 {
12910            assert!(matches!(
12911                on_child_exit(
12912                    &spec,
12913                    runtime.restart_policy,
12914                    &supervisor.registry,
12915                    &snapshot,
12916                    &runtime.terminal_ring,
12917                    &runtime.spawn_events,
12918                    &runtime.child_roster,
12919                    crash_exit_report(attempt),
12920                )
12921                .await,
12922                NextAction::Restart { schedule: _ }
12923            ));
12924        }
12925
12926        // The oldest crash moves out of the window; nothing else about the
12927        // module changes.
12928        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12929            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12930        })
12931        .unwrap();
12932
12933        assert!(
12934            matches!(
12935                on_child_exit(
12936                    &spec,
12937                    runtime.restart_policy,
12938                    &supervisor.registry,
12939                    &snapshot,
12940                    &runtime.terminal_ring,
12941                    &runtime.spawn_events,
12942                    &runtime.child_roster,
12943                    crash_exit_report(3),
12944                )
12945                .await,
12946                NextAction::Restart { schedule: _ }
12947            ),
12948            "a crash older than the window must not hold a budget slot"
12949        );
12950
12951        let state = lock_snapshot(&snapshot).unwrap();
12952        assert_eq!(state.state, ModuleState::Restarting);
12953        assert_eq!(
12954            state.crash_restarts.len(),
12955            2,
12956            "the aged instant is dropped and the new one takes its place"
12957        );
12958        assert_eq!(
12959            state.lifetime_restarts, 3,
12960            "the ledger counts every restart, including the ones the window forgot"
12961        );
12962    }
12963
12964    /// An operator restart hands the budget back whole, and the ledger keeps
12965    /// counting. Those are different questions -- "how close is this module to
12966    /// being stopped" and "how many times has it been replaced" -- and the
12967    /// operator action answers only the first.
12968    #[tokio::test]
12969    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12970        let supervisor = Supervisor::new_for_test(
12971            Arc::new(Registry::default()),
12972            RestartPolicy::new(2, Duration::ZERO),
12973        );
12974        let runtime = supervisor.runtime_config();
12975        let spec = windowed_crash_spec("operator-cleared-budget");
12976        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12977
12978        for attempt in 1..=2 {
12979            assert!(matches!(
12980                on_child_exit(
12981                    &spec,
12982                    runtime.restart_policy,
12983                    &supervisor.registry,
12984                    &snapshot,
12985                    &runtime.terminal_ring,
12986                    &runtime.spawn_events,
12987                    &runtime.child_roster,
12988                    crash_exit_report(attempt),
12989                )
12990                .await,
12991                NextAction::Restart { schedule: _ }
12992            ));
12993        }
12994
12995        reset_restart_count(&snapshot, &spec.module_id).unwrap();
12996        {
12997            let state = lock_snapshot(&snapshot).unwrap();
12998            assert!(
12999                state.crash_restarts.is_empty(),
13000                "an operator restart returns the full budget"
13001            );
13002            assert_eq!(
13003                state.lifetime_restarts, 2,
13004                "clearing the budget must not unmake the crashes"
13005            );
13006        }
13007
13008        assert!(
13009            matches!(
13010                on_child_exit(
13011                    &spec,
13012                    runtime.restart_policy,
13013                    &supervisor.registry,
13014                    &snapshot,
13015                    &runtime.terminal_ring,
13016                    &runtime.spawn_events,
13017                    &runtime.child_roster,
13018                    crash_exit_report(3),
13019                )
13020                .await,
13021                NextAction::Restart { schedule: _ }
13022            ),
13023            "the cleared budget must be spendable again"
13024        );
13025        let state = lock_snapshot(&snapshot).unwrap();
13026        assert_eq!(state.crash_restarts.len(), 1);
13027        assert_eq!(state.lifetime_restarts, 3);
13028    }
13029
13030    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13031    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13032        let severed = ProcessIdentity {
13033            pid: 41,
13034            start_time: 101,
13035        };
13036        let successor = ProcessIdentity {
13037            pid: 41,
13038            start_time: 202,
13039        };
13040        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13041        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13042            state.pid = Some(successor.pid);
13043            state.process_start_time = Some(successor.start_time);
13044        })
13045        .unwrap();
13046        assert!(!module.record_deliberate_severance(severed).unwrap());
13047
13048        let exit_report = apply_deliberate_severance_marker(
13049            &module.inner.snapshot,
13050            Some(successor),
13051            ExitReport {
13052                kind: ExitKind::Crash,
13053                code: Some(1),
13054                signal: None,
13055                at_ms: 1,
13056            },
13057        );
13058
13059        assert_eq!(exit_report.kind, ExitKind::Crash);
13060    }
13061
13062    #[tokio::test]
13063    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13064        let registry = Registry::default();
13065        let supervisor = Supervisor::new_for_test(
13066            Arc::new(Registry::default()),
13067            RestartPolicy::new(3, Duration::ZERO),
13068        );
13069        let runtime = supervisor.runtime_config();
13070        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13071        let spec = ModuleSpec {
13072            module_id: "drain-deliberate-severance".to_string(),
13073            program: fake_aft_stub_path(),
13074            args: Vec::new(),
13075            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13076            reserved: false,
13077            reserved_prefixes: Vec::new(),
13078            protocol: ModuleProtocol::Subc,
13079            overlap: Default::default(),
13080        };
13081        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13082        let process = ProcessIdentity {
13083            pid: 41,
13084            start_time: 101,
13085        };
13086        child.process_identity = Some(process);
13087        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13088            state.pid = Some(process.pid);
13089            state.process_start_time = Some(process.start_time);
13090        })
13091        .unwrap();
13092        record_deliberate_severance(&snapshot, process).unwrap();
13093
13094        drain_child_to_state(
13095            &spec.module_id,
13096            spec.protocol,
13097            // The child exits on its own; no signal may change the exit this
13098            // test classifies.
13099            StopNotice::SentOverConnection,
13100            &registry,
13101            None,
13102            &snapshot,
13103            &runtime.terminal_ring,
13104            &runtime.spawn_events,
13105            child,
13106            Duration::from_secs(1),
13107            ModuleState::Stopped,
13108            Some(false),
13109        )
13110        .await
13111        .unwrap();
13112
13113        let state = lock_snapshot(&snapshot).unwrap();
13114        assert_eq!(
13115            state.last_exit.as_ref().map(|exit| exit.kind),
13116            Some(ExitKind::DeliberateSeverance)
13117        );
13118        assert_eq!(state.lifetime_restarts, 1);
13119        assert_eq!(state.crash_restarts.len(), 0);
13120        drop(state);
13121        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13122        assert_eq!(
13123            history.entries[0].exit_kind,
13124            subc_control::TerminalExitKind::DeliberateSeverance
13125        );
13126    }
13127
13128    #[tokio::test]
13129    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13130        let registry = Registry::default();
13131        let supervisor = Supervisor::new_for_test(
13132            Arc::new(Registry::default()),
13133            RestartPolicy::new(3, Duration::ZERO),
13134        );
13135        let runtime = supervisor.runtime_config();
13136        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13137        let spec = ModuleSpec {
13138            module_id: "ordinary-drain".to_string(),
13139            program: fake_aft_stub_path(),
13140            args: Vec::new(),
13141            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13142            reserved: false,
13143            reserved_prefixes: Vec::new(),
13144            protocol: ModuleProtocol::Subc,
13145            overlap: Default::default(),
13146        };
13147        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13148
13149        drain_child_to_state(
13150            &spec.module_id,
13151            spec.protocol,
13152            // The child exits on its own; no signal may change the exit this
13153            // test classifies.
13154            StopNotice::SentOverConnection,
13155            &registry,
13156            None,
13157            &snapshot,
13158            &runtime.terminal_ring,
13159            &runtime.spawn_events,
13160            child,
13161            Duration::from_secs(1),
13162            ModuleState::Stopped,
13163            Some(false),
13164        )
13165        .await
13166        .unwrap();
13167
13168        let state = lock_snapshot(&snapshot).unwrap();
13169        assert_eq!(
13170            state.last_exit.as_ref().map(|exit| exit.kind),
13171            Some(ExitKind::Crash)
13172        );
13173        assert_eq!(state.lifetime_restarts, 0);
13174        assert_eq!(state.crash_restarts.len(), 0);
13175    }
13176
13177    #[test]
13178    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13179        // The server's generic fatal-routing branch only knows that the
13180        // connection failed; it does not know that the daemon deliberately
13181        // initiated a process-killing severance. Keep this seam explicit so a
13182        // future connection error path cannot silently reintroduce the stale
13183        // exemption that mislabels a later genuine crash.
13184        assert!(!include_str!("server.rs")
13185            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13186    }
13187
13188    /// The `route.closed` `drained` value must be the quiescence wait's own
13189    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13190    /// measurement at all and `false` is the one honest constant. This is the exact
13191    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13192    /// on every return path, including the one that used to return early via `?`
13193    /// with `route.closing` already sent and no `route.closed` ever following.
13194    #[test]
13195    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13196        assert!(drained_after_quiescence_wait(&Ok(true)));
13197        assert!(!drained_after_quiescence_wait(&Ok(false)));
13198        assert!(!drained_after_quiescence_wait(&Err(
13199            SuperviseError::StatePoisoned { module_id: None }
13200        )));
13201    }
13202
13203    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13204    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13205    /// already reaped out-of-band) still leaves a terminal record rather than none
13206    /// at all. Triggering the real `wait()` I/O error from an integration test would
13207    /// need a genuine already-reaped-child race, which is OS-specific and not
13208    /// something this suite attempts elsewhere; this test instead verifies the
13209    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13210    /// the call site itself is verified by inspection to sit in that exact arm.
13211    #[test]
13212    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13213        let ring = Arc::new(Mutex::new(TerminalRing::new(
13214            TerminalRingConfig::default(),
13215            0,
13216        )));
13217        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13218
13219        let snapshot = ring.lock().unwrap().snapshot();
13220        assert_eq!(snapshot.entries.len(), 1);
13221        let entry = &snapshot.entries[0];
13222        assert_eq!(entry.exit_code, None);
13223        assert_eq!(entry.exit_signal, None);
13224        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13225    }
13226
13227    #[test]
13228    fn wait_error_exit_path_preserves_spawn_event_density() {
13229        let feed = super::SpawnEventFeed::default();
13230        feed.configure_incarnation("wait-error-density".to_string());
13231        feed.emit_spawned("wait-error", 41, 1);
13232        let ring = Arc::new(Mutex::new(TerminalRing::new(
13233            TerminalRingConfig::default(),
13234            0,
13235        )));
13236
13237        record_wait_error_terminal("wait-error", &ring, &feed);
13238        feed.emit_spawned("after-wait-error", 42, 2);
13239
13240        let state = feed.0.lock().unwrap();
13241        let sequences = state
13242            .events
13243            .iter()
13244            .map(|event| event.cursor.seq)
13245            .collect::<Vec<_>>();
13246        assert_eq!(sequences, vec![1, 2, 3]);
13247        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13248        assert_eq!(state.events[1].exit_code, None);
13249        assert_eq!(state.events[1].exit_signal, None);
13250    }
13251
13252    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13253    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13254    /// not a clean exit it never actually observed.
13255    #[test]
13256    fn wait_error_exit_report_is_classified_as_a_crash() {
13257        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13258    }
13259}
13260
13261#[cfg(all(test, unix))]
13262mod restart_ordering_tests {
13263    use super::*;
13264    use std::io::Write;
13265    use tokio::io::AsyncReadExt;
13266
13267    #[test]
13268    #[ignore = "child process fixture"]
13269    fn serving_child() {
13270        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13271            return;
13272        };
13273        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13274        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13275        for stream in listener.incoming() {
13276            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13277        }
13278    }
13279
13280    async fn restart_before_publication(requests: usize) {
13281        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13282        let address_file = home.path().join("address");
13283        let spec = ModuleSpec {
13284            module_id: "restart-serving-child".into(),
13285            program: std::env::current_exe().unwrap(),
13286            args: vec![
13287                "--exact".into(),
13288                "supervise::restart_ordering_tests::serving_child".into(),
13289                "--ignored".into(),
13290                "--nocapture".into(),
13291            ],
13292            env: vec![(
13293                "RESTART_TEST_ADDRESS_FILE".into(),
13294                address_file.display().to_string(),
13295            )],
13296            reserved: false,
13297            reserved_prefixes: vec![],
13298            protocol: ModuleProtocol::None,
13299            overlap: Default::default(),
13300        };
13301        let mut supervisor =
13302            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13303        let (spawned_tx, spawned_rx) = oneshot::channel();
13304        let spawned_tx = Mutex::new(Some(spawned_tx));
13305        let (release_tx, release_rx) = std::sync::mpsc::channel();
13306        let release_rx = Mutex::new(release_rx);
13307        // Hold publication, not the child: it must answer a real request before
13308        // the restart enters the queue, even though status still has generation 0.
13309        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13310            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13311            release_rx
13312                .lock()
13313                .unwrap()
13314                .recv_timeout(Duration::from_secs(20))
13315                .unwrap();
13316        })));
13317        let registered = supervisor
13318            .register_configured_with_health(
13319                spec.clone(),
13320                true,
13321                HealthConfig::default(),
13322                None,
13323                RestartPolicy::new(3, Duration::ZERO),
13324            )
13325            .unwrap();
13326        let module = registered.module.clone();
13327        let startup = tokio::task::spawn_blocking(move || {
13328            supervisor.start_registered_configured(registered, None)
13329        });
13330        spawned_rx.await.unwrap();
13331        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13332            let address = loop {
13333                if let Ok(address) = std::fs::read_to_string(&address_file) {
13334                    if !address.is_empty() {
13335                        break address;
13336                    }
13337                }
13338                tokio::time::sleep(Duration::from_millis(1)).await;
13339            };
13340            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13341            let mut reply = String::new();
13342            stream.read_to_string(&mut reply).await.unwrap();
13343            reply.trim().parse::<u32>().unwrap()
13344        })
13345        .await
13346        .expect("child must serve before generation publication");
13347        assert_eq!(
13348            lock_snapshot(&module.inner.snapshot)
13349                .unwrap()
13350                .spawn_generation,
13351            0
13352        );
13353        let mut replies = Vec::new();
13354        for _ in 0..requests {
13355            let mut restart = Box::pin(module.restart(Some(0)));
13356            // Poll the public API through queue admission, leaving only its ACK
13357            // pending. The monitor cannot consume the command until released.
13358            std::future::poll_fn(|cx| {
13359                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13360                std::task::Poll::Ready(())
13361            })
13362            .await;
13363            replies.push(restart);
13364        }
13365        release_tx.send(()).unwrap();
13366        startup.await.unwrap().unwrap();
13367        for reply in replies {
13368            reply.await.unwrap();
13369        }
13370        // Restart schedules its replacement after completing the handler. Wait
13371        // for that separate actor turn before placing the command barrier.
13372        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13373            loop {
13374                if lock_snapshot(&module.inner.snapshot)
13375                    .unwrap()
13376                    .spawn_generation
13377                    > 1
13378                {
13379                    break;
13380                }
13381                tokio::time::sleep(Duration::from_millis(1)).await;
13382            }
13383        })
13384        .await;
13385        // Commands are handled serially. The update-spec reply proves every
13386        // restart handler has finished before we inspect the snapshot; restart's
13387        // own ACK only acknowledges initiation, not replacement completion.
13388        module.update_spec_for_test(spec).await.unwrap();
13389        let (generation, pid) = {
13390            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13391            (state.spawn_generation, state.pid)
13392        };
13393        module.stop().await.unwrap();
13394        replacement.expect("restart must publish a replacement for the already-serving child");
13395        assert_ne!(
13396            pid,
13397            Some(old_pid),
13398            "a serving child predating the restart must be replaced"
13399        );
13400        assert_eq!(generation, 2, "queued restarts must share one replacement");
13401    }
13402
13403    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13404    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13405        restart_before_publication(1).await;
13406    }
13407
13408    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13409    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13410        restart_before_publication(2).await;
13411    }
13412
13413    #[tokio::test]
13414    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13415        let supervisor =
13416            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13417        let registered = supervisor
13418            .register_configured_with_health(
13419                ModuleSpec {
13420                    module_id: "queued-before-replacement".into(),
13421                    program: "/bin/sh".into(),
13422                    args: vec!["-c".into(), "exec sleep 600".into()],
13423                    env: vec![],
13424                    reserved: false,
13425                    reserved_prefixes: vec![],
13426                    protocol: ModuleProtocol::None,
13427                    overlap: Default::default(),
13428                },
13429                true,
13430                HealthConfig::default(),
13431                None,
13432                RestartPolicy::default(),
13433            )
13434            .unwrap();
13435        let RegisteredConfiguredModule {
13436            mut spec,
13437            mut runtime,
13438            module,
13439            mut commands,
13440        } = registered;
13441        let snapshot = &module.inner.snapshot;
13442        let mut restart = Box::pin(module.restart(Some(0)));
13443        std::future::poll_fn(|cx| {
13444            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13445            std::task::Poll::Ready(())
13446        })
13447        .await;
13448        // Drive the actor explicitly so a replacement is definitely spawned
13449        // after receipt, but before this queued command is dequeued.
13450        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13451        let replacement_pid = child.as_ref().unwrap().id();
13452        let command = commands.recv().await.unwrap();
13453        assert!(
13454            handle_supervisor_command(
13455                command,
13456                &mut spec,
13457                &mut runtime,
13458                &supervisor.registry,
13459                &supervisor.process_liveness,
13460                snapshot,
13461                &mut child,
13462                &mut commands,
13463                &mut VecDeque::new(),
13464            )
13465            .await
13466        );
13467        restart.await.unwrap();
13468        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13469        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13470        if let Some(mut child) = child {
13471            child.start_kill().unwrap();
13472            child.wait().await.unwrap();
13473        }
13474        assert_eq!(
13475            kept_pid, replacement_pid,
13476            "a newer replacement must survive"
13477        );
13478        assert!(
13479            !pending,
13480            "a satisfied restart must not schedule another spawn"
13481        );
13482    }
13483}
13484
13485#[cfg(all(test, unix))]
13486mod health_event_tests {
13487    use super::*;
13488    use cortexkit_test_support::ScratchDir;
13489    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13490
13491    struct Actor {
13492        module: SupervisedModule,
13493        registry: Arc<Registry>,
13494        forwarding: Arc<ForwardingTable>,
13495        spec: ModuleSpec,
13496        health: HealthConfig,
13497        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13498        _home: ScratchDir,
13499    }
13500
13501    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13502        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13503        // for an observed condition, rather than letting paused time auto-advance
13504        // deadlines or assuming a fixed number of yields finishes the work.
13505        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13506        let tick = Instant::now();
13507        loop {
13508            if let Some(value) = observe() {
13509                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13510                return value;
13511            }
13512            assert!(
13513                std::time::Instant::now() < deadline,
13514                "timed out waiting for {reason}"
13515            );
13516            tokio::task::yield_now().await;
13517        }
13518    }
13519
13520    impl Actor {
13521        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13522            let home = ScratchDir::new("health-events");
13523            let registry = Arc::new(Registry::default());
13524            let forwarding = Arc::new(ForwardingTable::default());
13525            let health = HealthConfig {
13526                cadence,
13527                deadline: Duration::from_secs(3600),
13528                ..HealthConfig::default()
13529            };
13530            let supervisor =
13531                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13532                    .with_forwarding(forwarding.clone())
13533                    .with_health_config(health.clone());
13534            let spec = ModuleSpec {
13535                module_id: "event-health-child".into(),
13536                program: PathBuf::from("/bin/sh"),
13537                // The process stays alive but has no real bus peer. Registrations
13538                // below exercise the same registry writes as accepted HELLOs.
13539                args: vec!["-c".into(), "exec sleep 600".into()],
13540                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13541                    .into_iter()
13542                    .map(|key| (key.into(), home.path().display().to_string()))
13543                    .collect(),
13544                reserved: false,
13545                reserved_prefixes: vec![],
13546                protocol,
13547                overlap: Default::default(),
13548            };
13549            let module = supervisor.spawn(spec.clone()).unwrap();
13550            let (_, rx) = mpsc::channel(8);
13551            let actor = Self {
13552                module,
13553                registry,
13554                forwarding,
13555                spec,
13556                health,
13557                rx,
13558                _home: home,
13559            };
13560            actor
13561                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13562                .await;
13563            actor
13564        }
13565
13566        fn turns(&self) -> u64 {
13567            lock_snapshot(&self.module.inner.snapshot)
13568                .unwrap()
13569                .actor_turns
13570        }
13571
13572        async fn wait_for_select(
13573            &self,
13574            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13575        ) -> ActorSelectCheckpoint {
13576            wait_for("actor select checkpoint", || {
13577                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13578                state.actor_select.filter(|checkpoint| {
13579                    checkpoint.generation == state.spawn_generation
13580                        && state.state == ModuleState::Running
13581                        && state.process_alive
13582                        && state.reported_pid().is_some()
13583                        && expected(checkpoint)
13584                })
13585            })
13586            .await
13587        }
13588
13589        fn hello(&mut self, connection: u64, health: bool) {
13590            let manifest =
13591                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13592                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13593                    .build();
13594            let connection = ConnectionId::new(connection);
13595            let (tx, rx) = mpsc::channel(8);
13596            self.rx = rx;
13597            self.forwarding
13598                .register_module_connection(
13599                    connection,
13600                    self.spec.module_id.clone(),
13601                    subc_protocol::PROTOCOL_VERSION,
13602                    Concurrency::ModuleManaged,
13603                    FrameSink::new(tx),
13604                )
13605                .unwrap();
13606            self.registry
13607                .register_with_control_ops(
13608                    manifest,
13609                    subc_protocol::PROTOCOL_VERSION,
13610                    connection,
13611                    if health {
13612                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13613                    } else {
13614                        vec![]
13615                    },
13616                )
13617                .unwrap();
13618        }
13619
13620        async fn probe(&mut self) -> crate::router::OutboundFrame {
13621            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13622                Ok(frame) => Some(frame),
13623                Err(mpsc::error::TryRecvError::Empty) => None,
13624                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13625            })
13626            .await;
13627            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13628            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13629            frame
13630        }
13631
13632        fn no_probe(&mut self) {
13633            assert!(matches!(
13634                self.rx.try_recv(),
13635                Err(mpsc::error::TryRecvError::Empty)
13636            ));
13637        }
13638
13639        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13640            self.forwarding
13641                .complete_module_control_rpc(
13642                    ConnectionId::new(connection),
13643                    frame.header.corr,
13644                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13645                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13646                        status: HealthStatus::Ok,
13647                        detail: None,
13648                        metrics: None,
13649                    }),
13650                )
13651                .unwrap();
13652        }
13653    }
13654
13655    #[tokio::test(start_paused = true)]
13656    async fn no_health_children_park_for_sixty_seconds() {
13657        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13658        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13659        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13660        unadvertised.hello(1, false);
13661        // Include any one-off registration wake in the budget; receiving a
13662        // no-health HELLO must not turn a parked child into a polling child.
13663        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13664        // a single 60 s jump would only observe one expired timer.
13665        for _ in 0..6000 {
13666            tokio::time::advance(Duration::from_millis(10)).await;
13667            tokio::task::yield_now().await;
13668        }
13669        for actor in [&none, &unregistered, &unadvertised] {
13670            assert!(
13671                actor.turns() <= 5,
13672                "no-health actor woke {} times",
13673                actor.turns()
13674            );
13675            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13676        }
13677    }
13678
13679    #[tokio::test(start_paused = true)]
13680    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13681        // With a zero cadence the first probe is due at once, so this measures
13682        // only how fast the registration notification starts health probing,
13683        // not the normal 30-33 s wait before a first probe (checked below).
13684        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13685        let tick = Instant::now();
13686        actor.hello(1, true);
13687        actor.probe().await;
13688        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13689    }
13690
13691    #[tokio::test(start_paused = true)]
13692    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13693        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13694        let hello_tick = Instant::now();
13695        actor.hello(1, true);
13696        let armed = actor
13697            .wait_for_select(|checkpoint| {
13698                checkpoint.registered_connection == Some(ConnectionId::new(1))
13699                    && checkpoint.next_probe_at.is_some()
13700            })
13701            .await;
13702        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13703        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13704        let first_deadline = hello_tick + delay;
13705        assert_eq!(armed.next_probe_at, Some(first_deadline));
13706        tokio::time::advance(Duration::from_secs(29)).await;
13707        let turns = actor.turns();
13708        actor
13709            .registry
13710            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13711            .unwrap();
13712        let updated = actor
13713            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13714            .await;
13715        assert_eq!(
13716            updated.next_probe_at,
13717            Some(first_deadline),
13718            "catalog update must not reset cadence"
13719        );
13720        actor.no_probe();
13721        tokio::time::advance(first_deadline - Instant::now()).await;
13722        let frame = actor.probe().await;
13723        actor.answer(1, frame);
13724        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13725        let next_deadline = Instant::now() + next;
13726        let rearmed = actor
13727            .wait_for_select(|checkpoint| {
13728                checkpoint
13729                    .next_probe_at
13730                    .is_some_and(|deadline| deadline > first_deadline)
13731            })
13732            .await;
13733        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13734        tokio::time::advance(next - Duration::from_nanos(1)).await;
13735        actor.no_probe();
13736        tokio::time::advance(Duration::from_nanos(1)).await;
13737        actor.probe().await;
13738    }
13739
13740    #[tokio::test(start_paused = true)]
13741    async fn disconnect_disarms_and_reconnect_rearms_health() {
13742        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13743        actor.hello(1, true);
13744        actor
13745            .wait_for_select(|checkpoint| {
13746                checkpoint.registered_connection == Some(ConnectionId::new(1))
13747                    && checkpoint.next_probe_at.is_some()
13748            })
13749            .await;
13750        let turns = actor.turns();
13751        let tick = Instant::now();
13752        actor
13753            .registry
13754            .deregister_connection(ConnectionId::new(1))
13755            .unwrap();
13756        let disarmed = actor
13757            .wait_for_select(|checkpoint| {
13758                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13759            })
13760            .await;
13761        assert_eq!(disarmed.next_probe_at, None);
13762        assert_eq!(disarmed.wake_after, None);
13763        assert_eq!(
13764            Instant::now(),
13765            tick,
13766            "disconnect must disarm in the notification tick"
13767        );
13768        tokio::time::advance(Duration::from_secs(60)).await;
13769        actor.no_probe();
13770        let reconnect_tick = Instant::now();
13771        actor.hello(2, true);
13772        let rearmed = actor
13773            .wait_for_select(|checkpoint| {
13774                checkpoint.registered_connection == Some(ConnectionId::new(2))
13775                    && checkpoint.next_probe_at.is_some()
13776            })
13777            .await;
13778        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13779        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13780        tokio::time::advance(delay).await;
13781        actor.probe().await;
13782    }
13783
13784    #[tokio::test(start_paused = true)]
13785    async fn reregistration_without_health_disarms_the_old_schedule() {
13786        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13787        actor.hello(1, true);
13788        actor
13789            .wait_for_select(|checkpoint| {
13790                checkpoint.registered_connection == Some(ConnectionId::new(1))
13791                    && checkpoint.next_probe_at.is_some()
13792            })
13793            .await;
13794        actor
13795            .registry
13796            .deregister_connection(ConnectionId::new(1))
13797            .unwrap();
13798        actor.hello(2, false);
13799        let disarmed = actor
13800            .wait_for_select(|checkpoint| {
13801                checkpoint.registered_connection == Some(ConnectionId::new(2))
13802            })
13803            .await;
13804        assert_eq!(disarmed.next_probe_at, None);
13805        assert_eq!(disarmed.wake_after, None);
13806        tokio::time::advance(Duration::from_secs(60)).await;
13807        actor.no_probe();
13808    }
13809
13810    #[tokio::test(start_paused = true)]
13811    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13812        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13813        actor.hello(1, true);
13814        let armed = actor
13815            .wait_for_select(|checkpoint| {
13816                checkpoint.registered_connection == Some(ConnectionId::new(1))
13817                    && checkpoint.next_probe_at.is_some()
13818            })
13819            .await;
13820        // Simulate the old process's connection closing before the restart
13821        // tears the process down. The replacement process has started but has
13822        // not sent its HELLO, so it is not registered.
13823        actor
13824            .registry
13825            .deregister_connection(ConnectionId::new(1))
13826            .unwrap();
13827        actor.module.restart(Some(0)).await.unwrap();
13828        let parked = actor
13829            .wait_for_select(|checkpoint| {
13830                checkpoint.generation > armed.generation
13831                    && checkpoint.registered_connection.is_none()
13832                    && checkpoint.next_probe_at.is_none()
13833            })
13834            .await;
13835        assert_eq!(parked.wake_after, None);
13836        actor.rx.close();
13837        // Drain the stop notice the restart queued for the old process's connection,
13838        // so it isn't mistaken for traffic to the replacement.
13839        while actor.rx.try_recv().is_ok() {}
13840        let turns = actor.turns();
13841        tokio::time::advance(Duration::from_secs(60)).await;
13842        let after = actor.turns();
13843        eprintln!(
13844            "restart park turns: before={turns} after={after} delta={}",
13845            after - turns
13846        );
13847        assert_eq!(after, turns, "replacement must park before HELLO");
13848        let hello_tick = Instant::now();
13849        actor.hello(2, true);
13850        let rearmed = actor
13851            .wait_for_select(|checkpoint| {
13852                checkpoint.registered_connection == Some(ConnectionId::new(2))
13853                    && checkpoint.next_probe_at.is_some()
13854            })
13855            .await;
13856        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13857        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13858        tokio::time::advance(delay).await;
13859        actor.probe().await;
13860    }
13861
13862    #[tokio::test(start_paused = true)]
13863    async fn rescan_adds_and_removes_http_health_without_polling() {
13864        use tokio::{
13865            io::{AsyncReadExt, AsyncWriteExt},
13866            net::TcpListener,
13867        };
13868        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13869        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13870        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13871        // One millisecond separates successive probes without a polling delay.
13872        actor.health.cadence = Duration::from_millis(1);
13873        let (started_tx, mut started_rx) = mpsc::channel(8);
13874        let peer = tokio::spawn(async move {
13875            loop {
13876                let (mut socket, _) = listener.accept().await.unwrap();
13877                let mut request = [0; 1024];
13878                assert!(socket.read(&mut request).await.unwrap() > 0);
13879                started_tx.send(()).await.unwrap();
13880                socket
13881                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13882                    .await
13883                    .unwrap();
13884            }
13885        });
13886        let tick = Instant::now();
13887        let turns = actor.turns();
13888        actor
13889            .module
13890            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13891            .await
13892            .unwrap();
13893        let armed = actor
13894            .wait_for_select(|checkpoint| {
13895                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13896            })
13897            .await;
13898        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13899        assert_eq!(Instant::now(), tick);
13900        tokio::time::advance(Duration::from_millis(1)).await;
13901        let mut started = false;
13902        wait_for("successful HTTP probe", || {
13903            started |= started_rx.try_recv().is_ok();
13904            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13905                .then_some(())
13906        })
13907        .await;
13908        assert!(started, "added HTTP check must start at its first deadline");
13909        assert_eq!(
13910            actor.module.status().unwrap().health.status,
13911            SupervisorHealthStatus::Ok
13912        );
13913        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13914        actor.health.http = None;
13915        let turns = actor.turns();
13916        actor
13917            .module
13918            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13919            .await
13920            .unwrap();
13921        let parked = actor
13922            .wait_for_select(|checkpoint| {
13923                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13924            })
13925            .await;
13926        assert_eq!(parked.wake_after, None);
13927        let turns = actor.turns();
13928        tokio::time::advance(Duration::from_secs(60)).await;
13929        assert_eq!(
13930            actor.turns(),
13931            turns,
13932            "removed HTTP check must park the actor"
13933        );
13934        assert!(
13935            started_rx.try_recv().is_err(),
13936            "removed HTTP check must stay disarmed"
13937        );
13938        assert_eq!(
13939            actor.module.status().unwrap().health,
13940            ModuleHealthStatus::default()
13941        );
13942        peer.abort();
13943    }
13944}
13945
13946#[cfg(test)]
13947mod drain_busy_reprobe_tests {
13948    //! The drain's busy-gauge wait, driven against an in-process module
13949    //! connection whose health answers the test scripts. Time is paused, so
13950    //! every elapsed duration below is exact virtual time.
13951    use super::*;
13952    use std::collections::BTreeMap;
13953    use std::sync::Mutex as StdMutex;
13954    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13955
13956    const MODULE_ID: &str = "drain-reprobe-module";
13957    const CONNECTION: u64 = 7;
13958    const GAUGE: &str = "runs_in_flight";
13959    /// The production default for both the health cadence and the drain
13960    /// ceiling, which is the pairing that used to cost a full ceiling.
13961    const CADENCE: Duration = Duration::from_secs(30);
13962    const CEILING: Duration = Duration::from_secs(30);
13963
13964    /// How the scripted module answers each `health.check`.
13965    #[derive(Clone, Copy)]
13966    enum Answer {
13967        /// Report the gauge busy until this instant, quiet from then on, after
13968        /// taking `latency` to answer.
13969        BusyUntil { until: Instant, latency: Duration },
13970        /// Answer at once, with metrics that leave the declared gauge out.
13971        OmitsGauge,
13972        /// Never answer.
13973        Silent,
13974    }
13975
13976    struct Drain {
13977        forwarding: Arc<ForwardingTable>,
13978        runtime: SupervisorRuntimeConfig,
13979        endpoint: crate::ModuleEndpointId,
13980        /// When each probe reached the module.
13981        probes: Arc<StdMutex<Vec<Instant>>>,
13982        responder: tokio::task::JoinHandle<()>,
13983    }
13984
13985    impl Drain {
13986        fn start(answer: Answer) -> Self {
13987            let registry = Arc::new(Registry::default());
13988            let forwarding = Arc::new(ForwardingTable::default());
13989            let health = HealthConfig {
13990                cadence: CADENCE,
13991                ..HealthConfig::default()
13992            };
13993            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13994                .with_forwarding(forwarding.clone())
13995                .with_health_config(health)
13996                .runtime_config();
13997            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13998            let endpoint = forwarding
13999                .register_module_connection(
14000                    ConnectionId::new(CONNECTION),
14001                    MODULE_ID.to_string(),
14002                    subc_protocol::PROTOCOL_VERSION,
14003                    Concurrency::ModuleManaged,
14004                    FrameSink::new(tx),
14005                )
14006                .unwrap();
14007            let probes = Arc::new(StdMutex::new(Vec::new()));
14008            let responder = tokio::spawn({
14009                let forwarding = forwarding.clone();
14010                let probes = probes.clone();
14011                async move {
14012                    while let Some(frame) = rx.recv().await {
14013                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14014                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14015                        probes.lock().unwrap().push(Instant::now());
14016                        let metrics = match answer {
14017                            Answer::BusyUntil { until, latency } => {
14018                                sleep(latency).await;
14019                                let busy = u64::from(Instant::now() < until);
14020                                serde_json::json!({ GAUGE: busy })
14021                            }
14022                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14023                            Answer::Silent => continue,
14024                        };
14025                        // A probe abandoned at its deadline has no RPC left to
14026                        // complete, so a late answer may be refused.
14027                        let _ = forwarding.complete_module_control_rpc(
14028                            ConnectionId::new(CONNECTION),
14029                            frame.header.corr,
14030                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14031                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14032                                status: HealthStatus::Ok,
14033                                detail: None,
14034                                metrics: Some(metrics),
14035                            }),
14036                        );
14037                    }
14038                }
14039            });
14040            Self {
14041                forwarding,
14042                runtime,
14043                endpoint,
14044                probes,
14045                responder,
14046            }
14047        }
14048
14049        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14050        /// how long it took.
14051        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14052            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14053            (drained, elapsed)
14054        }
14055
14056        /// [`Self::run`], also returning what the drain's probes read.
14057        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14058            let started = Instant::now();
14059            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14060            let drained = wait_for_forwarding_quiescence(
14061                &self.forwarding,
14062                MODULE_ID,
14063                &self.runtime,
14064                self.endpoint,
14065                started + ceiling,
14066                &[GAUGE.to_string()],
14067                DrainScope::Active,
14068                &mut trace,
14069            )
14070            .await
14071            .unwrap();
14072            (drained, started.elapsed(), trace)
14073        }
14074
14075        fn probes(&self) -> Vec<Instant> {
14076            self.probes.lock().unwrap().clone()
14077        }
14078    }
14079
14080    impl Drop for Drain {
14081        fn drop(&mut self) {
14082            self.responder.abort();
14083        }
14084    }
14085
14086    #[tokio::test(start_paused = true)]
14087    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14088        let drain = Drain::start(Answer::BusyUntil {
14089            until: Instant::now() + Duration::from_secs(1),
14090            latency: Duration::ZERO,
14091        });
14092        let (drained, elapsed) = drain.run(CEILING).await;
14093        assert!(
14094            drained,
14095            "gauges that read zero during the drain must let it finish drained, \
14096             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14097        );
14098        assert!(
14099            elapsed < Duration::from_secs(3),
14100            "drain took {elapsed:?} after the gauges went quiet at 1s"
14101        );
14102        let probes = drain.probes();
14103        assert!(
14104            probes.len() >= 2,
14105            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14106        );
14107    }
14108
14109    /// The same shape in real time on a multi-threaded runtime: the first
14110    /// probe is answered busy at once, within a millisecond of the drain
14111    /// starting, and later probes read idle. The drain must keep probing
14112    /// after that first instant answer and end soon after the gauge falls.
14113    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14114    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14115        let drain = Drain::start(Answer::BusyUntil {
14116            until: Instant::now() + Duration::from_millis(600),
14117            latency: Duration::ZERO,
14118        });
14119        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14120        assert!(drained, "the gauge fell at 600ms: {trace}");
14121        assert!(
14122            elapsed < Duration::from_secs(3),
14123            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14124        );
14125        let probes = drain.probes();
14126        assert!(
14127            probes.len() >= 2 && trace.probes as usize == probes.len(),
14128            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14129        );
14130    }
14131
14132    #[tokio::test(start_paused = true)]
14133    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14134        let started = Instant::now();
14135        let drain = Drain::start(Answer::BusyUntil {
14136            until: started + Duration::from_secs(3600),
14137            latency: Duration::ZERO,
14138        });
14139        let (drained, elapsed) = drain.run(CEILING).await;
14140        assert!(!drained, "a module busy throughout must not drain");
14141        assert!(
14142            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14143            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14144        );
14145        // Re-probing is paced by the interval, not a busy loop: at most one
14146        // probe per interval across the ceiling, plus the first at time zero.
14147        let probes = drain.probes();
14148        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14149        assert!(
14150            probes.len() as u128 <= max_probes,
14151            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14152            probes.len()
14153        );
14154        for pair in probes.windows(2) {
14155            assert!(
14156                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14157                "consecutive probes {:?} apart",
14158                pair[1] - pair[0]
14159            );
14160        }
14161    }
14162
14163    #[tokio::test(start_paused = true)]
14164    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14165        // Each answer takes longer than the re-probe interval; the next probe
14166        // must wait for the previous answer and then the interval.
14167        let latency = Duration::from_secs(1);
14168        let drain = Drain::start(Answer::BusyUntil {
14169            until: Instant::now() + Duration::from_secs(3600),
14170            latency,
14171        });
14172        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14173        assert!(!drained);
14174        let probes = drain.probes();
14175        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14176        for pair in probes.windows(2) {
14177            assert!(
14178                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14179                "a probe started {:?} after the previous one, before it could finish",
14180                pair[1] - pair[0]
14181            );
14182        }
14183    }
14184
14185    #[tokio::test(start_paused = true)]
14186    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14187        let drain = Drain::start(Answer::Silent);
14188        let ceiling = Duration::from_secs(2);
14189        let (drained, elapsed) = drain.run(ceiling).await;
14190        assert!(!drained, "an unanswered probe counts as busy");
14191        assert!(
14192            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14193            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14194        );
14195        assert_eq!(
14196            drain.probes().len(),
14197            1,
14198            "the unanswered probe holds the only slot until the ceiling"
14199        );
14200    }
14201
14202    /// Every event logged while it is installed, as `message` plus fields.
14203    #[derive(Clone, Default)]
14204    struct LogCapture {
14205        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14206    }
14207
14208    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14209        fn on_event(
14210            &self,
14211            event: &tracing::Event<'_>,
14212            _context: tracing_subscriber::layer::Context<'_, S>,
14213        ) {
14214            struct Fields(BTreeMap<String, String>);
14215            impl tracing::field::Visit for Fields {
14216                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14217                    self.0
14218                        .insert(field.name().to_string(), format!("{value:?}"));
14219                }
14220            }
14221            let mut fields = Fields(BTreeMap::new());
14222            event.record(&mut fields);
14223            self.events.lock().unwrap().push(fields.0);
14224        }
14225    }
14226
14227    impl LogCapture {
14228        fn install(&self) -> tracing::subscriber::DefaultGuard {
14229            use tracing_subscriber::layer::SubscriberExt;
14230            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14231        }
14232
14233        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14234            self.events
14235                .lock()
14236                .unwrap()
14237                .iter()
14238                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14239                .cloned()
14240                .collect()
14241        }
14242    }
14243
14244    /// A module that declares a busy gauge but leaves it out of its health
14245    /// answers holds a drain to its ceiling. The timeout line must say so by
14246    /// name, and the drain must warn about the missing gauge once, not on
14247    /// every probe.
14248    #[tokio::test(start_paused = true)]
14249    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14250        let capture = LogCapture::default();
14251        let _guard = capture.install();
14252        let drain = Drain::start(Answer::OmitsGauge);
14253        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14254        assert!(!drained, "a gauge never reported cannot read quiet");
14255        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14256
14257        let missing = capture.with_message("lacks a declared busy gauge");
14258        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14259        assert_eq!(
14260            missing[0].get("missing_gauges").map(String::as_str),
14261            Some(GAUGE)
14262        );
14263
14264        warn_drain_timed_out(
14265            MODULE_ID,
14266            Duration::from_secs(2),
14267            RouteCloseReason::Restart,
14268            &Default::default(),
14269            &trace,
14270        );
14271        let timeouts = capture.with_message("route drain timed out");
14272        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14273        let gauges = &timeouts[0]["busy_gauges"];
14274        assert!(
14275            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14276            "the timeout line must name the gauge and why it read busy: {gauges}"
14277        );
14278        assert!(
14279            gauges.ends_with("; no value ever read"),
14280            "no value was ever read for the gauge: {gauges}"
14281        );
14282        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14283    }
14284
14285    /// The trace names a busy value and when it was read. For a probe that
14286    /// never got an answer, it names the probe's error kind and says that no
14287    /// value was ever read.
14288    #[tokio::test(start_paused = true)]
14289    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14290        let busy = Drain::start(Answer::BusyUntil {
14291            until: Instant::now() + Duration::from_secs(3600),
14292            latency: Duration::ZERO,
14293        });
14294        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14295        let rendered = trace.to_string();
14296        assert!(
14297            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14298                && rendered.contains("; last value non_zero(1) @"),
14299            "{rendered}"
14300        );
14301
14302        let silent = Drain::start(Answer::Silent);
14303        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14304        // The probe is capped at the 1 s ceiling, and its reading is stamped
14305        // when it gave up.
14306        assert_eq!(
14307            trace.to_string(),
14308            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14309        );
14310        assert_eq!(trace.probes, 1);
14311    }
14312
14313    /// A gauge value that is not a count reads busy and is shown as sent.
14314    #[test]
14315    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14316        let gauges = [GAUGE.to_string()];
14317        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14318        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14319        assert_eq!(
14320            busy_gauge_observation(&verdicts),
14321            BusyGaugeObservation::Busy
14322        );
14323        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14324        assert_eq!(
14325            busy_gauge_observation(&zero),
14326            BusyGaugeObservation::Quiescent
14327        );
14328        let none = gauge_verdicts(None, &gauges);
14329        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14330    }
14331}
14332
14333#[cfg(test)]
14334mod health_evidence_tests {
14335    use super::{HealthProbeError, HealthProbeEvidence};
14336    use std::collections::HashSet;
14337
14338    /// The evidential asymmetry, asserted rather than described.
14339    ///
14340    /// Exactly ONE observation is proof a module cannot serve, and the one that
14341    /// fires under CPU starvation is not it. Before the split, all fifteen
14342    /// construction sites collapsed into a single String, so a timeout carried the
14343    /// same weight as a dead lane -- which is how a healthy module was restarted
14344    /// three times in one day.
14345    #[test]
14346    fn only_a_dead_lane_is_proof_of_death() {
14347        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14348        // Three non-proof classes, each for a different reason: silence is
14349        // consistent with health, a bad answer proves the module ALIVE, and a
14350        // daemon-side fault never reached the module at all.
14351        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14352        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14353        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14354    }
14355
14356    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14357    ///
14358    /// A shared label renders two different observations identically in the line an
14359    /// operator reads after an unexplained restart -- the exact confusion this
14360    /// change removes.
14361    #[test]
14362    fn every_evidence_class_has_a_distinct_label() {
14363        let labels = [
14364            HealthProbeError::lane_dead("").label(),
14365            HealthProbeError::no_answer("").label(),
14366            HealthProbeError::bad_answer("").label(),
14367            HealthProbeError::misconfigured("").label(),
14368        ];
14369        let unique: HashSet<_> = labels.iter().collect();
14370        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14371    }
14372
14373    /// The class is additional information, not a replacement.
14374    ///
14375    /// An operator needs both "this was silence" and the specific text saying how
14376    /// long we waited; a classification that swallowed the message would trade one
14377    /// missing distinction for another.
14378    #[test]
14379    fn classification_preserves_the_original_message() {
14380        let err = HealthProbeError::no_answer("module did not answer within 5s");
14381        assert_eq!(err.to_string(), "module did not answer within 5s");
14382        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14383    }
14384}
14385
14386#[cfg(test)]
14387mod health_tombstone_tests {
14388    use std::{path::PathBuf, sync::Arc, time::Duration};
14389
14390    use subc_protocol::{
14391        manifest::Concurrency,
14392        session::{HealthStatus, ModuleControlResponse},
14393    };
14394    use tokio::sync::mpsc;
14395
14396    use super::{
14397        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14398        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14399    };
14400    use crate::{
14401        control::ControlHandler,
14402        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14403        registry::{ConnectionId, Registry},
14404        router::FrameSink,
14405    };
14406
14407    struct ProbeHarness {
14408        spec: ModuleSpec,
14409        runtime: SupervisorRuntimeConfig,
14410        forwarding: Arc<ForwardingTable>,
14411        module_connection: ConnectionId,
14412        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14413        handler: ControlHandler,
14414        module: super::SupervisedModule,
14415    }
14416
14417    fn probe_harness() -> ProbeHarness {
14418        let registry = Arc::new(Registry::default());
14419        let forwarding = Arc::new(ForwardingTable::default());
14420        let supervisor_handle = super::SupervisorHandle::new();
14421        let health = HealthConfig {
14422            http: None,
14423            cadence: Duration::from_secs(30),
14424            deadline: Duration::from_secs(5),
14425            failure_threshold: 3,
14426            on_degraded: HealthAction::Report,
14427            on_failing: HealthAction::Report,
14428            critical: false,
14429        };
14430        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14431            .with_forwarding(Arc::clone(&forwarding))
14432            .with_handle(supervisor_handle.clone())
14433            .with_health_config(health);
14434        let spec = ModuleSpec {
14435            module_id: "late-health-module".to_string(),
14436            program: PathBuf::from("disabled-module"),
14437            args: Vec::new(),
14438            env: Vec::new(),
14439            reserved: false,
14440            reserved_prefixes: Vec::new(),
14441            protocol: ModuleProtocol::Subc,
14442            overlap: Default::default(),
14443        };
14444        let module = supervisor
14445            .supervise_configured(spec.clone(), false)
14446            .unwrap();
14447        let runtime = supervisor.runtime_config();
14448        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14449            .with_supervisor(supervisor_handle);
14450        let module_connection = ConnectionId::new(700);
14451        let (module_tx, module_rx) = mpsc::channel(8);
14452        forwarding
14453            .register_module_connection(
14454                module_connection,
14455                spec.module_id.clone(),
14456                subc_protocol::PROTOCOL_VERSION,
14457                Concurrency::ModuleManaged,
14458                FrameSink::new(module_tx),
14459            )
14460            .unwrap();
14461
14462        ProbeHarness {
14463            spec,
14464            runtime,
14465            forwarding,
14466            module_connection,
14467            module_rx,
14468            handler,
14469            module,
14470        }
14471    }
14472
14473    async fn finish_after(
14474        harness: &mut ProbeHarness,
14475        stall: Duration,
14476    ) -> ModuleControlRpcCompletion {
14477        assert!(stall > harness.runtime.health.deadline);
14478        let deadline = harness.runtime.health.deadline;
14479        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14480        let answer = async {
14481            let frame = harness.module_rx.recv().await.expect("health.check frame");
14482            tokio::time::advance(deadline).await;
14483            tokio::task::yield_now().await;
14484            tokio::time::advance(stall - deadline).await;
14485            harness
14486                .forwarding
14487                .complete_module_control_rpc(
14488                    harness.module_connection,
14489                    frame.header.corr,
14490                    Some("health.check"),
14491                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14492                        status: HealthStatus::Ok,
14493                        detail: None,
14494                        metrics: None,
14495                    }),
14496                )
14497                .unwrap()
14498        };
14499        let (probe_result, completion) = tokio::join!(probe, answer);
14500        let err = probe_result.expect_err("probe must miss its deadline");
14501        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14502        completion
14503    }
14504
14505    async fn time_out_without_answer(harness: &mut ProbeHarness) {
14506        let deadline = harness.runtime.health.deadline;
14507        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14508        let exhaust_deadline = async {
14509            let _frame = harness.module_rx.recv().await.expect("health.check frame");
14510            tokio::time::advance(deadline).await;
14511            tokio::task::yield_now().await;
14512        };
14513        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14514        let err = probe_result.expect_err("probe must miss its deadline");
14515        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14516    }
14517
14518    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14519        let registry = Arc::clone(&harness.module.inner.registry);
14520        let snapshot = Arc::clone(&harness.module.inner.snapshot);
14521        let process_liveness = super::SupervisorProcessLiveness::default();
14522        let mut child = None;
14523        let cycle = super::run_health_probe_cycle(
14524            &harness.spec,
14525            &harness.runtime,
14526            &registry,
14527            &process_liveness,
14528            &snapshot,
14529            &mut child,
14530        );
14531        let peer = async {
14532            let frame = harness.module_rx.recv().await.expect("health.check frame");
14533            if answer {
14534                harness
14535                    .forwarding
14536                    .complete_module_control_rpc(
14537                        harness.module_connection,
14538                        frame.header.corr,
14539                        Some("health.check"),
14540                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14541                            status: HealthStatus::Ok,
14542                            detail: None,
14543                            metrics: Some(serde_json::json!({"ready": true})),
14544                        }),
14545                    )
14546                    .unwrap();
14547            } else {
14548                tokio::time::advance(harness.runtime.health.deadline).await;
14549                tokio::task::yield_now().await;
14550            }
14551        };
14552        tokio::join!(cycle, peer);
14553    }
14554
14555    #[tokio::test(start_paused = true)]
14556    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14557        let mut harness = probe_harness();
14558        // Drive the probe cycle directly with an in-memory wire peer. Stop the
14559        // disabled module's monitor so only this test owns lifecycle transitions;
14560        // no OS process is launched, and a restart is observed at scheduling.
14561        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14562        monitor.abort();
14563        let _ = monitor.await;
14564        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14565            state.enabled = true;
14566            state.state = super::ModuleState::Running;
14567            state.process_alive = true;
14568        })
14569        .unwrap();
14570
14571        run_probe_cycle(&mut harness, true).await;
14572        assert_eq!(
14573            harness.module.status().unwrap().health.status,
14574            super::SupervisorHealthStatus::Ok
14575        );
14576
14577        for failures in 1..harness.runtime.health.failure_threshold {
14578            run_probe_cycle(&mut harness, false).await;
14579            let status = harness.module.status().unwrap();
14580            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14581            assert_eq!(status.health.consecutive_failures, failures);
14582            assert!(status.health.last_probe_ms.is_some());
14583            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14584            assert!(status.health.metrics.is_none());
14585            assert_eq!(status.state, super::ModuleState::Running);
14586            assert!(status.process_alive);
14587            assert_eq!(status.restart_count, 0);
14588            assert_eq!(status.lifetime_restarts, 0);
14589            assert!(status.health.last_action.is_none());
14590            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14591        }
14592
14593        run_probe_cycle(&mut harness, true).await;
14594        let recovered = harness.module.status().unwrap();
14595        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14596        assert_eq!(recovered.health.consecutive_failures, 0);
14597        assert!(recovered.health.detail.is_none());
14598        assert_eq!(
14599            recovered.health.metrics,
14600            Some(serde_json::json!({"ready": true}))
14601        );
14602        assert_eq!(recovered.lifetime_restarts, 0);
14603
14604        for failures in 1..=harness.runtime.health.failure_threshold {
14605            run_probe_cycle(&mut harness, false).await;
14606            let status = harness.module.status().unwrap();
14607            assert_eq!(status.health.consecutive_failures, failures);
14608            if failures < harness.runtime.health.failure_threshold {
14609                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14610                assert_eq!(status.state, super::ModuleState::Running);
14611                assert_eq!(status.lifetime_restarts, 0);
14612                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14613            } else {
14614                assert_eq!(
14615                    status.health.status,
14616                    super::SupervisorHealthStatus::Unresponsive
14617                );
14618                assert_eq!(status.state, super::ModuleState::Restarting);
14619                assert_eq!(status.restart_count, 1);
14620                assert_eq!(status.lifetime_restarts, 1);
14621                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14622                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14623            }
14624        }
14625    }
14626
14627    #[tokio::test(start_paused = true)]
14628    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14629        let mut harness = probe_harness();
14630
14631        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14632        let first_latency = match &first {
14633            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14634            other => panic!("late answer was not retained: {other:?}"),
14635        };
14636        assert!(harness.handler.observe_module_control_completion(first));
14637
14638        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14639        let second_latency = match &second {
14640            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14641            other => panic!("late answer was not retained: {other:?}"),
14642        };
14643        assert!(harness.handler.observe_module_control_completion(second));
14644
14645        assert_eq!(first_latency, Duration::from_secs(8));
14646        assert_eq!(
14647            second_latency - first_latency,
14648            Duration::from_secs(3),
14649            "latency must grow linearly with the additional stall"
14650        );
14651        let health = harness.module.status().unwrap().health;
14652        assert_eq!(health.late_answer_count, 2);
14653        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14654    }
14655
14656    /// A module that answers every probe late must never march to the kill
14657    /// threshold: the late answer proves it is alive, so it must clear the miss
14658    /// streak the timeout recorded. Without the reset, a CPU-starved module
14659    /// that serves every probe seconds past the deadline accumulates
14660    /// `consecutive_failures` to the threshold and is killed — the exact
14661    /// sequence from the 2026-08-14 aft disable, where the daemon logged
14662    /// "proves the module is alive" five times while counting five misses.
14663    #[tokio::test(start_paused = true)]
14664    async fn late_answer_clears_the_consecutive_failure_streak() {
14665        let mut harness = probe_harness();
14666
14667        // Timeout recorded first: the probe path saw no answer in time.
14668        time_out_without_answer(&mut harness).await;
14669        harness
14670            .module
14671            .record_health_probe_failure_for_test("[no-answer] test miss")
14672            .unwrap();
14673        assert_eq!(
14674            harness.module.status().unwrap().health.consecutive_failures,
14675            1,
14676            "precondition: the miss must be on the streak before the late answer"
14677        );
14678
14679        // The stalled reply then lands: proof of life.
14680        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14681        assert!(matches!(
14682            late,
14683            ModuleControlRpcCompletion::LateHealthAnswer { .. }
14684        ));
14685        assert!(harness.handler.observe_module_control_completion(late));
14686
14687        let health = harness.module.status().unwrap().health;
14688        assert_eq!(
14689            health.consecutive_failures, 0,
14690            "a late answer is an answer: the streak must reset"
14691        );
14692        assert_eq!(health.late_answer_count, 1);
14693    }
14694
14695    #[tokio::test(start_paused = true)]
14696    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14697        let mut harness = probe_harness();
14698
14699        for _ in 0..20 {
14700            time_out_without_answer(&mut harness).await;
14701            assert_eq!(
14702                harness.forwarding.health_probe_tombstone_count().unwrap(),
14703                1
14704            );
14705        }
14706    }
14707}
14708
14709#[cfg(test)]
14710mod child_env_tests {
14711    #[cfg(windows)]
14712    use super::{
14713        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14714    };
14715    #[cfg(windows)]
14716    use std::sync::{Arc, OnceLock};
14717    #[cfg(windows)]
14718    #[test]
14719    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14720        let spec = spec(Vec::new());
14721        let mut command = Command::new("unused");
14722        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14723            .unwrap()
14724            .unwrap();
14725        let name = command
14726            .as_std()
14727            .get_envs()
14728            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14729            .and_then(|(_, value)| value)
14730            .unwrap();
14731        assert_eq!(name, OsStr::new(handoff.name()));
14732        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14733        assert!(command.as_std().get_envs().any(|(key, value)| key
14734            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14735            && value == Some(OsStr::new("env"))));
14736        assert!(command
14737            .as_std()
14738            .get_envs()
14739            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14740    }
14741
14742    #[cfg(windows)]
14743    #[test]
14744    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14745        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14746        snapshot.process_alive = true;
14747        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14748        assert_eq!(
14749            observed_launch_nonce_source(&snapshot).as_deref(),
14750            Some("env")
14751        );
14752        let consumed = Arc::new(OnceLock::new());
14753        snapshot.pipe_consumed = Some(consumed.clone());
14754        assert_eq!(
14755            observed_launch_nonce_source(&snapshot).as_deref(),
14756            Some("env")
14757        );
14758        consumed.set(()).unwrap();
14759        assert_eq!(
14760            observed_launch_nonce_source(&snapshot).as_deref(),
14761            Some("pipe")
14762        );
14763        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14764        assert_eq!(
14765            observed_launch_nonce_source(&snapshot).as_deref(),
14766            Some("env")
14767        );
14768        clear_current_process_facts(&mut snapshot);
14769        assert_eq!(observed_launch_nonce_source(&snapshot), None);
14770    }
14771    use super::{
14772        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14773        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14774        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14775    };
14776    use std::{ffi::OsStr, path::PathBuf};
14777    use tokio::process::Command;
14778
14779    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14780        ModuleSpec {
14781            module_id: "env-plan".to_string(),
14782            program: PathBuf::from("/nonexistent"),
14783            args: Vec::new(),
14784            env,
14785            reserved: false,
14786            reserved_prefixes: Vec::new(),
14787            protocol: ModuleProtocol::Subc,
14788            overlap: Default::default(),
14789        }
14790    }
14791
14792    #[test]
14793    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14794        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14795        let digest = LaunchSpecDigest::from_spec(&base);
14796        let mut reordered = base.clone();
14797        reordered.env.reverse();
14798        reordered.module_id = "different-id".into();
14799        reordered.reserved = true;
14800        reordered.overlap = Default::default();
14801        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14802        let mut delimiter_collision = base.clone();
14803        delimiter_collision.env[0] = ("A=B".into(), "C".into());
14804        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14805        let mut changed = base.clone();
14806        changed.protocol = ModuleProtocol::None;
14807        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14808        changed = base.clone();
14809        changed.args.push("--flag".into());
14810        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14811        changed = base.clone();
14812        changed.program = PathBuf::from("/other");
14813        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14814        changed = base.clone();
14815        changed.env[1].1 = "other".into();
14816        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14817        assert!(!format!("{digest:?}").contains("B=C"));
14818        assert!(!format!("{digest:?}").contains("[u8"));
14819    }
14820
14821    #[cfg(unix)]
14822    #[test]
14823    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14824        use std::os::unix::ffi::OsStringExt;
14825        let mut left = spec(Vec::new());
14826        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14827        let mut right = left.clone();
14828        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14829        assert_eq!(
14830            left.program.to_string_lossy(),
14831            right.program.to_string_lossy()
14832        );
14833        assert_ne!(
14834            LaunchSpecDigest::from_spec(&left),
14835            LaunchSpecDigest::from_spec(&right)
14836        );
14837    }
14838
14839    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
14840    /// one still gets its own.
14841    ///
14842    /// This is the narrow goal `env_clear()` was reached for, and the reason the
14843    /// fix is `env_remove` rather than deleting the line: an operator's ambient
14844    /// filter silently becoming an unconfigured module's log level is a real
14845    /// defect, just a much smaller one than clearing the environment.
14846    ///
14847    /// Asserted on the command plan rather than a spawned child because proving
14848    /// the ABSENCE of an inherited variable needs the parent's environment
14849    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
14850    /// removal as `(key, None)`, which is exactly the distinction wanted: not
14851    /// "absent because nobody set it" but "explicitly unset for the child".
14852    #[test]
14853    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14854        let mut command = Command::new("/nonexistent");
14855        apply_child_env(&mut command, &spec(Vec::new()));
14856        let removed = command
14857            .as_std()
14858            .get_envs()
14859            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14860        assert!(
14861            removed,
14862            "ambient CK_LOG must be explicitly removed for an unconfigured module"
14863        );
14864
14865        let mut configured = Command::new("/nonexistent");
14866        apply_child_env(
14867            &mut configured,
14868            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14869        );
14870        let effective = configured
14871            .as_std()
14872            .get_envs()
14873            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14874            .last()
14875            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14876        assert_eq!(
14877            effective,
14878            Some(Some("debug".to_string())),
14879            "a module's configured CK_LOG must survive the ambient removal"
14880        );
14881    }
14882
14883    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
14884    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
14885    /// the same reason as the CK_LOG test above.
14886    ///
14887    /// The argument is the load-bearing half: a stock binary exits on an
14888    /// unknown flag before it listens, so with `--subc` appended the mode
14889    /// could not supervise the one process it exists for. Found by the first
14890    /// conformance run (nats-server: `flag provided but not defined: -subc`).
14891    #[test]
14892    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14893        let connection_file = std::path::Path::new("/run/subc-connection.json");
14894        let handle = SupervisorHandle::new();
14895
14896        let mut none_spec = spec(Vec::new());
14897        none_spec.protocol = ModuleProtocol::None;
14898        let mut none = Command::new("/nonexistent");
14899        let none_handoff =
14900            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14901                .expect("protocol-none spawn args apply");
14902        assert!(
14903            none_handoff.is_none(),
14904            "protocol:none spawn must not receive a nonce descriptor"
14905        );
14906        #[cfg(windows)]
14907        assert!(!none.as_std().get_envs().any(|(key, value)| key
14908            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14909            && value.is_some()));
14910        assert!(
14911            !none.as_std().get_envs().any(|(key, value)| key
14912                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14913                && value.is_some()),
14914            "protocol:none spawn must not name a nonce descriptor"
14915        );
14916        let none_args: Vec<String> = none
14917            .as_std()
14918            .get_args()
14919            .map(|a| a.to_string_lossy().into_owned())
14920            .collect();
14921        assert!(
14922            !none_args.iter().any(|a| a == SUBC_ARG),
14923            "protocol:none argv must not carry --subc; got {none_args:?}"
14924        );
14925        let none_has_nonce = none
14926            .as_std()
14927            .get_envs()
14928            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14929        assert!(
14930            !none_has_nonce,
14931            "protocol:none spawn must not receive a launch nonce"
14932        );
14933        let none_has_module_id = none
14934            .as_std()
14935            .get_envs()
14936            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14937        assert!(
14938            none_has_module_id,
14939            "SUBC_MODULE_ID is inert and stays on every path"
14940        );
14941        assert!(
14942            handle.spawn_nonce(&none_spec.module_id).is_none(),
14943            "no nonce record for a process that will never present one"
14944        );
14945
14946        // Control: the subc-wire path is unchanged by the branch above.
14947        let wire_spec = spec(Vec::new());
14948        let mut wire = Command::new("/nonexistent");
14949        let wire_handoff =
14950            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14951                .expect("subc-wire spawn args apply");
14952        let wire_fd_env = wire
14953            .as_std()
14954            .get_envs()
14955            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14956            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14957        #[cfg(unix)]
14958        assert_eq!(
14959            wire_fd_env,
14960            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14961            "a subc-wire spawn names the pipe it will receive at descriptor 3"
14962        );
14963        #[cfg(windows)]
14964        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14965        #[cfg(not(any(unix, windows)))]
14966        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14967        let wire_args: Vec<String> = wire
14968            .as_std()
14969            .get_args()
14970            .map(|a| a.to_string_lossy().into_owned())
14971            .collect();
14972        assert_eq!(
14973            wire_args,
14974            vec![
14975                SUBC_ARG.to_string(),
14976                connection_file.to_string_lossy().into_owned()
14977            ],
14978            "a subc-wire spawn still carries --subc <path>"
14979        );
14980        assert_eq!(
14981            wire.as_std()
14982                .get_envs()
14983                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14984            !cfg!(unix),
14985            "only Windows supplies the environment nonce"
14986        );
14987        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14988    }
14989
14990    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
14991    /// spec tries to set it; only a swap candidate carries it.
14992    ///
14993    /// "Set it only on candidates" is not enough, because spawn applies the
14994    /// spec's env verbatim and the daemon's own environment is inherited: either
14995    /// could hand a plain restart the swap role, and a module reading it would
14996    /// warm on its long swap budget while callers wait. Asserted as an explicit
14997    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
14998    /// test above gives.
14999    #[test]
15000    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15001        let role = |command: &Command| {
15002            command
15003                .as_std()
15004                .get_envs()
15005                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15006                .last()
15007                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15008        };
15009        let forged = spec(vec![(
15010            SUBC_SPAWN_ROLE_ENV.to_string(),
15011            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15012        )]);
15013
15014        let mut plain = Command::new("/nonexistent");
15015        apply_child_env(&mut plain, &forged);
15016        apply_spawn_role(&mut plain, SpawnRole::Plain);
15017        assert_eq!(
15018            role(&plain),
15019            Some(None),
15020            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15021        );
15022
15023        let mut candidate = Command::new("/nonexistent");
15024        apply_child_env(&mut candidate, &spec(Vec::new()));
15025        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15026        assert_eq!(
15027            role(&candidate),
15028            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15029        );
15030    }
15031
15032    /// Daemon-private capture retention keys never reach the child.
15033    ///
15034    /// cortexkit-log exposes retention as a Rust struct with no environment
15035    /// names, so these entries are supervisor metadata. Passing them through
15036    /// would invent a public child-process contract by accident.
15037    #[test]
15038    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15039        let mut command = Command::new("/nonexistent");
15040        apply_child_env(
15041            &mut command,
15042            &spec(vec![
15043                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15044                ("KEPT".to_string(), "yes".to_string()),
15045            ]),
15046        );
15047        let keys: Vec<String> = command
15048            .as_std()
15049            .get_envs()
15050            .filter(|(_, value)| value.is_some())
15051            .map(|(key, _)| key.to_string_lossy().into_owned())
15052            .collect();
15053        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15054        assert!(
15055            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15056            "daemon-private capture key leaked to the child: {keys:?}"
15057        );
15058    }
15059}
15060
15061#[cfg(test)]
15062mod jitter_tests {
15063    use super::jittered_health_delay;
15064    use std::{collections::HashSet, time::Duration};
15065
15066    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15067    /// that actually occur rather than invented ones.
15068    ///
15069    /// This is a SAMPLE, not a registry: the property under test is that distinct
15070    /// ids disperse, which holds for any set of distinct strings. Several entries
15071    /// are already historical (modules get renamed), and that costs nothing here --
15072    /// but it means a reader must not mistake this for the live module set, and a
15073    /// rename sweep will match it without there being anything to change.
15074    const FLEET: [&str; 14] = [
15075        "aft",
15076        "alfonso-core",
15077        "magic-context",
15078        "broca",
15079        "thalamus",
15080        "quota",
15081        "engram",
15082        "plexus",
15083        "cerebellum",
15084        "astrocyte",
15085        "synapse",
15086        "subc-mcp",
15087        "cortexkit-credentials",
15088        "subc-federation",
15089    ];
15090
15091    /// Probes must not converge after a fleet-wide restart.
15092    ///
15093    /// This is the property the jitter exists for: every module reconnects at
15094    /// once, and without dispersal all fourteen would then probe on the same
15095    /// tick forever. Nothing failed visibly when this went untested -- a
15096    /// convergent fleet still probes correctly, just in a burst, so the symptom
15097    /// is a periodic load spike that looks like whatever else is running.
15098    #[test]
15099    fn probe_delays_disperse_across_the_fleet() {
15100        let cadence = Duration::from_secs(30);
15101        let delays: HashSet<Duration> = FLEET
15102            .iter()
15103            .map(|id| jittered_health_delay(id, 0, cadence))
15104            .collect();
15105        assert_eq!(
15106            delays.len(),
15107            FLEET.len(),
15108            "every supervised module must land on its own probe offset"
15109        );
15110    }
15111
15112    /// The offset may only ever DELAY a probe, never bring it forward.
15113    ///
15114    /// A delay below the cadence would probe a module more often than
15115    /// configured, which is the opposite of what an operator asked for and
15116    /// would tighten the failure budget without anyone changing it.
15117    #[test]
15118    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15119        let cadence = Duration::from_secs(30);
15120        let span = cadence / 10;
15121        for id in FLEET {
15122            for probe_index in 0..8 {
15123                let delay = jittered_health_delay(id, probe_index, cadence);
15124                assert!(
15125                    delay >= cadence,
15126                    "{id}#{probe_index}: jitter must not shorten the cadence"
15127                );
15128                assert!(
15129                    delay < cadence + span,
15130                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15131                );
15132            }
15133        }
15134    }
15135
15136    /// A module keeps its offset across daemon restarts.
15137    ///
15138    /// The delay is derived rather than randomised precisely so a restart does
15139    /// not re-roll every module into a fresh chance of collision. A random
15140    /// source would satisfy the dispersal test above and quietly lose this.
15141    #[test]
15142    fn a_module_offset_is_stable_across_restarts() {
15143        let cadence = Duration::from_secs(30);
15144        for id in FLEET {
15145            assert_eq!(
15146                jittered_health_delay(id, 0, cadence),
15147                jittered_health_delay(id, 0, cadence),
15148                "{id}: the same module and probe index must produce the same offset"
15149            );
15150        }
15151    }
15152
15153    /// A zero cadence disables probing rather than producing a busy loop.
15154    #[test]
15155    fn zero_cadence_yields_zero_delay() {
15156        assert_eq!(
15157            jittered_health_delay("aft", 0, Duration::ZERO),
15158            Duration::ZERO
15159        );
15160    }
15161}
15162
15163#[cfg(all(test, target_os = "linux"))]
15164mod cgroup_placement_tests {
15165    use super::{
15166        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15167        SuperviseError, SupervisedChild,
15168    };
15169    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15170    use cortexkit_test_support::ScratchDir;
15171    use std::{
15172        fs, io,
15173        path::{Path, PathBuf},
15174        sync::{Arc, Mutex},
15175    };
15176    use tokio::process::Command;
15177
15178    #[tokio::test]
15179    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15180        use super::*;
15181        let dir = ScratchDir::new("unique-spawn-cgroups");
15182        let root = PathBuf::from(format!(
15183            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15184            std::process::id(),
15185            unix_ms_now()
15186        ));
15187        if let Err(error) = fs::create_dir(&root) {
15188            assert!(
15189                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15190                "required cgroup test cannot execute: {error}"
15191            );
15192            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15193            return;
15194        }
15195        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15196        let group_count = || {
15197            fs::read_dir(root.join("subc-modules"))
15198                .unwrap()
15199                .map(|entry| entry.unwrap().file_type().unwrap())
15200                .filter(|kind| kind.is_dir())
15201                .count()
15202        };
15203        let supervisor =
15204            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15205                .with_cgroup_placement(Some(placement.clone()));
15206        let runtime = supervisor.runtime_config();
15207        let mut spec = ModuleSpec {
15208            module_id: "unique-spawn".into(),
15209            program: PathBuf::from("/bin/sleep"),
15210            args: vec!["60".into()],
15211            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15212                .into_iter()
15213                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15214                .collect(),
15215            reserved: false,
15216            reserved_prefixes: vec![],
15217            protocol: ModuleProtocol::None,
15218            overlap: Default::default(),
15219        };
15220        let spawn = |spec: &ModuleSpec| {
15221            spawn_child(
15222                spec,
15223                None,
15224                None,
15225                &runtime.stderr_ring,
15226                None,
15227                &runtime.child_roster,
15228                Some(&placement),
15229            )
15230            .unwrap()
15231        };
15232        let mut live = spawn(&spec);
15233        for _ in 0..3 {
15234            // A new process can enter the old slot while retirement is pending.
15235            let next = spawn(&spec);
15236            assert_ne!(live.module_id, next.module_id);
15237            live.start_kill().unwrap();
15238            live.wait().await.unwrap();
15239            live = next;
15240            assert!(
15241                live.child.try_wait().unwrap().is_none(),
15242                "retiring the old slot must not kill the replacement"
15243            );
15244            assert_eq!(
15245                group_count(),
15246                1,
15247                "only the live spawn's cgroup should remain"
15248            );
15249        }
15250        supervisor.begin_daemon_shutdown();
15251        let reap = tokio::spawn(async move {
15252            live.wait().await.unwrap();
15253        });
15254        supervisor
15255            .end_children_for_daemon_shutdown(false, std::future::pending())
15256            .await;
15257        reap.await.unwrap();
15258        assert_eq!(group_count(), 0);
15259        // A normal exit uses the same tree-cleanup path as a killed spawn.
15260        spec.program = PathBuf::from("/bin/true");
15261        spec.args.clear();
15262        let fresh_roster = ChildRoster::default();
15263        let mut short = spawn_child(
15264            &spec,
15265            None,
15266            None,
15267            &runtime.stderr_ring,
15268            None,
15269            &fresh_roster,
15270            Some(&placement),
15271        )
15272        .unwrap();
15273        short.wait().await.unwrap();
15274        assert_eq!(group_count(), 0);
15275        spec.module_id = "_".repeat(255);
15276        let mut long_id = spawn_child(
15277            &spec,
15278            None,
15279            None,
15280            &runtime.stderr_ring,
15281            None,
15282            &fresh_roster,
15283            Some(&placement),
15284        )
15285        .unwrap();
15286        long_id.wait().await.unwrap();
15287        assert_eq!(
15288            group_count(),
15289            0,
15290            "valid long module IDs must not exceed cgroup NAME_MAX"
15291        );
15292        fs::remove_dir(root.join("subc-modules")).unwrap();
15293        fs::remove_dir(root).unwrap();
15294        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15295    }
15296
15297    #[test]
15298    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15299        let path = Path::new("/definitely-missing-subc-cgroup");
15300        let mut command = Command::new("true");
15301        let error = apply_cgroup_placement(
15302            &mut command,
15303            &ModuleSpec {
15304                module_id: "broken-cgroup".to_string(),
15305                program: PathBuf::from("true"),
15306                args: Vec::new(),
15307                env: Vec::new(),
15308                reserved: false,
15309                reserved_prefixes: Vec::new(),
15310                protocol: ModuleProtocol::Subc,
15311                overlap: Default::default(),
15312            },
15313            path,
15314        )
15315        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15316        let reason = error.to_string();
15317
15318        assert!(
15319            matches!(error, SuperviseError::Cgroup { .. }),
15320            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15321        );
15322        assert!(
15323            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15324            "parent cgroup open failure must name cgroup.procs: {reason}"
15325        );
15326    }
15327
15328    #[tokio::test]
15329    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15330        let root = ScratchDir::new("supervisor-reap-cgroup");
15331        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15332        let placement = subc_cgroup::prepare_at(&root)
15333            .expect("prepare scratch cgroup root")
15334            .expect("scratch root has a cgroup.procs marker");
15335        let module_id = "reaped-module";
15336        let module = placement
15337            .module_path(module_id)
15338            .expect("create scratch module cgroup");
15339        let child = Command::new("true")
15340            .env("XDG_DATA_HOME", root.path())
15341            .env("XDG_RUNTIME_DIR", root.path())
15342            .env("XDG_CONFIG_HOME", root.path())
15343            .spawn()
15344            .expect("spawn short-lived child");
15345        let pid = child.id().expect("spawned child has pid");
15346        let mut child = SupervisedChild {
15347            child,
15348            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15349            protocol: ModuleProtocol::Subc,
15350            module_id: module_id.to_string(),
15351            cgroup_placement: Some(placement),
15352            stdout_pump: None,
15353            stderr_pump: None,
15354            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15355            spawned_at: tokio::time::Instant::now(),
15356            spawned_at_ms: 0,
15357            spawned_from: PathBuf::from("true"),
15358            launch_spec_digest: LaunchSpecDigest([0; 32]),
15359            spawned_file_identity: None,
15360            process_start_time: None,
15361            process_identity: None,
15362            pid,
15363            roster_guard: None,
15364            #[cfg(target_os = "macos")]
15365            privacy_exec: None,
15366            spawn_failure: None,
15367        };
15368
15369        child.wait().await.expect("reap short-lived child");
15370
15371        assert!(
15372            !module.exists(),
15373            "reaping the supervised child must remove its empty cgroup"
15374        );
15375    }
15376
15377    #[test]
15378    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15379        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15380        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15381        let placement = subc_cgroup::prepare_at(&root)
15382            .expect("prepare scratch cgroup root")
15383            .expect("scratch root has a cgroup.procs marker");
15384        let module = placement
15385            .module_path("surviving-module")
15386            .expect("create scratch module cgroup");
15387        fs::write(module.join("surviving-process"), b"still present")
15388            .expect("make scratch cgroup non-empty");
15389        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15390
15391        remove_module_cgroup(&placement, "surviving-module");
15392
15393        let logs = crate::router::test_log::captured_logs(&logs);
15394        assert!(
15395            module.exists(),
15396            "failed removal must leave the cgroup intact"
15397        );
15398        assert!(
15399            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15400                && logs.contains("surviving-module"),
15401            "best-effort removal must report the failure without returning it: {logs}"
15402        );
15403    }
15404
15405    #[test]
15406    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15407        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15408        let reason = SuperviseError::Spawn {
15409            program: PathBuf::from("/bin/true"),
15410            source: io::Error::from_raw_os_error(13),
15411            cgroup_path: Some(cgroup_path.clone()),
15412        }
15413        .to_string();
15414
15415        assert!(
15416            reason.contains(&cgroup_path.display().to_string()),
15417            "a pre_exec spawn failure must name the cgroup path: {reason}"
15418        );
15419    }
15420}
15421
15422#[cfg(test)]
15423mod spawn_subscriber_lag_tests {
15424    use super::*;
15425
15426    /// A subscriber whose connection stops draining is dropped once its frame
15427    /// channel fills. The client must learn that from a terminal Error frame
15428    /// after the frames already queued for it, not from a stream that simply
15429    /// goes quiet.
15430    #[tokio::test]
15431    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15432        let feed = SpawnEventFeed::default();
15433        feed.configure_incarnation("lag-incarnation".to_string());
15434        // A one-slot connection queue that nobody reads until the emits are
15435        // done: the forwarder parks on it and the subscriber channel fills.
15436        let (tx, mut rx) = mpsc::channel(1);
15437        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15438            .expect("subscribe");
15439        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15440        for index in 0..emitted {
15441            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15442            // Let the forwarder take what it can so the fill point is the
15443            // subscriber channel, not a scheduling accident.
15444            tokio::task::yield_now().await;
15445        }
15446        assert_eq!(
15447            feed.subscriber_count(),
15448            0,
15449            "the lagged subscriber must be removed"
15450        );
15451
15452        let mut data = Vec::new();
15453        let mut last = None;
15454        loop {
15455            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15456                .await
15457                .expect("the forwarder must finish once the subscriber is dropped");
15458            let Some(outbound) = next else { break };
15459            let frame = outbound.frame;
15460            if frame.header.ty == FrameType::StreamData {
15461                assert!(last.is_none(), "no data may follow the terminal frame");
15462                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15463                data.push(event.cursor.seq);
15464            } else {
15465                assert!(last.is_none(), "exactly one terminal frame");
15466                last = Some(frame);
15467            }
15468        }
15469        assert!(!data.is_empty(), "queued frames drain before the terminal");
15470        for pair in data.windows(2) {
15471            assert_eq!(
15472                pair[1],
15473                pair[0] + 1,
15474                "queued frames arrive dense and in order"
15475            );
15476        }
15477        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15478        assert_eq!(terminal.header.ty, FrameType::Error);
15479        assert_eq!(terminal.header.corr, 7);
15480        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15481        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15482        let detail = body.detail.expect("lagged error carries detail");
15483        assert_eq!(
15484            detail["first_undelivered_cursor"]["seq"],
15485            data.last().unwrap() + 1,
15486            "the named cursor is the first event the subscriber did not receive"
15487        );
15488        assert_eq!(
15489            detail["first_undelivered_cursor"]["daemon_incarnation"],
15490            "lag-incarnation"
15491        );
15492    }
15493}
15494
15495#[cfg(test)]
15496mod terminal_history_read_concurrency_tests {
15497    use super::*;
15498    use crate::terminal_journal::read_pause;
15499    use cortexkit_test_support::ScratchDir;
15500    use std::sync::mpsc as std_mpsc;
15501
15502    fn journaled_ring(
15503        journal: &Arc<crate::terminal_journal::TerminalJournal>,
15504    ) -> Arc<Mutex<TerminalRing>> {
15505        Arc::new(Mutex::new(
15506            TerminalRing::new(TerminalRingConfig::default(), 1)
15507                .with_journal(Some(Arc::clone(journal))),
15508        ))
15509    }
15510
15511    fn crash(at_ms: u64) -> ExitReport {
15512        ExitReport {
15513            kind: ExitKind::Crash,
15514            code: Some(1),
15515            signal: None,
15516            at_ms,
15517        }
15518    }
15519
15520    /// Record an exit on another thread and report whether it finished within
15521    /// `bound`. The recorder thread is left running if it did not.
15522    fn record_within(
15523        module_id: &'static str,
15524        ring: &Arc<Mutex<TerminalRing>>,
15525        at_ms: u64,
15526        bound: Duration,
15527    ) -> bool {
15528        let ring = Arc::clone(ring);
15529        let (done, done_rx) = std_mpsc::channel();
15530        std::thread::spawn(move || {
15531            record_terminal(
15532                module_id,
15533                &ring,
15534                &SpawnEventFeed::default(),
15535                &crash(at_ms),
15536                TerminalDisposition::Restarting,
15537            );
15538            let _ = done.send(());
15539        });
15540        done_rx.recv_timeout(bound).is_ok()
15541    }
15542
15543    /// A history read in progress must not hold the journal writer (which every
15544    /// module's exit recording needs) or the module's own ring. Exits recorded
15545    /// while the read is paused complete promptly; the paused read answers as of
15546    /// the moment it started, and the next read has each exit exactly once.
15547    #[test]
15548    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15549        let dir = ScratchDir::new("terminal-history-concurrent-read");
15550        let path = dir.join("terminals.jsonl");
15551        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15552            path.clone(),
15553            "daemon".into(),
15554        ));
15555        let reader_ring = journaled_ring(&journal);
15556        let other_ring = journaled_ring(&journal);
15557        assert!(record_within(
15558            "reader-module",
15559            &reader_ring,
15560            10,
15561            Duration::from_secs(5)
15562        ));
15563
15564        let (started, release) = read_pause::install(&path);
15565        let reading = {
15566            let ring = Arc::clone(&reader_ring);
15567            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15568        };
15569        started
15570            .recv_timeout(Duration::from_secs(5))
15571            .expect("the history read reached its pause");
15572
15573        let bound = Duration::from_secs(1);
15574        assert!(
15575            record_within("other-module", &other_ring, 20, bound),
15576            "another module's exit waited on a history read (journal writer held)"
15577        );
15578        assert!(
15579            record_within("reader-module", &reader_ring, 30, bound),
15580            "the read module's own exit waited on its history read (ring held)"
15581        );
15582
15583        drop(release);
15584        let paused = reading.join().unwrap();
15585        assert_eq!(
15586            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15587            vec![10],
15588            "an exit recorded after the read began lands in neither half of it"
15589        );
15590        assert_eq!(paused.journal_skipped_lines, 0);
15591        assert_eq!(paused.journal_read_errors, 0);
15592
15593        let after = durable_terminal_history_of(&reader_ring, "reader-module");
15594        assert_eq!(
15595            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15596            vec![10, 30],
15597            "the next read merges ring and journal with no duplicate"
15598        );
15599        assert_eq!(after.journal_skipped_lines, 0);
15600    }
15601}
15602
15603/// What a restart does with the exited process's output readers. These drive
15604/// the same settle functions the supervisor calls, with a reader the test
15605/// holds, so a reader that has not been scheduled by the bound is a controlled
15606/// input rather than something only a loaded machine produces.
15607#[cfg(test)]
15608mod stderr_settle_tests {
15609    use std::{
15610        future::Future,
15611        io,
15612        pin::Pin,
15613        sync::{Arc, Mutex},
15614        task::{Context, Poll},
15615        time::Duration,
15616    };
15617
15618    use tokio::{
15619        io::{AsyncRead, ReadBuf},
15620        sync::oneshot,
15621        time::Instant,
15622    };
15623
15624    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15625    use crate::stderr_tail::{
15626        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15627        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15628    };
15629
15630    const BOUND: Duration = Duration::from_millis(250);
15631
15632    /// Yields `before`, then stays pending until the gate is released, then
15633    /// yields `after` and reaches EOF. The bytes after the gate were written
15634    /// by a process that has already exited; only the reader is behind.
15635    struct HeldReader {
15636        before: Option<Vec<u8>>,
15637        gate: Option<oneshot::Receiver<()>>,
15638        after: io::Cursor<Vec<u8>>,
15639    }
15640
15641    impl AsyncRead for HeldReader {
15642        fn poll_read(
15643            mut self: Pin<&mut Self>,
15644            cx: &mut Context<'_>,
15645            buf: &mut ReadBuf<'_>,
15646        ) -> Poll<io::Result<()>> {
15647            if let Some(bytes) = self.before.take() {
15648                buf.put_slice(&bytes);
15649                return Poll::Ready(Ok(()));
15650            }
15651            if let Some(gate) = self.gate.as_mut() {
15652                match Pin::new(gate).poll(cx) {
15653                    Poll::Pending => return Poll::Pending,
15654                    Poll::Ready(_) => self.gate = None,
15655                }
15656            }
15657            Pin::new(&mut self.after).poll_read(cx, buf)
15658        }
15659    }
15660
15661    struct DiscardSink;
15662
15663    impl OutputSink for DiscardSink {
15664        fn write_line(&mut self, _line: &[u8]) {}
15665    }
15666
15667    fn line(text: &str) -> TailEntry {
15668        TailEntry::Line {
15669            text: text.to_string(),
15670            truncated: false,
15671            at_ms: None,
15672        }
15673    }
15674
15675    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15676        ring.lock().unwrap()
15677    }
15678
15679    /// Start a reader for a new process generation that delivers `before`
15680    /// immediately and `after` only once the returned sender fires (or is
15681    /// dropped).
15682    fn held_pump(
15683        ring: &Arc<Mutex<StderrRing>>,
15684        before: &str,
15685        after: &str,
15686    ) -> (StderrPump, oneshot::Sender<()>) {
15687        let generation = lock(ring).begin_process();
15688        let (release, gate) = oneshot::channel();
15689        let reader = HeldReader {
15690            before: Some(before.as_bytes().to_vec()),
15691            gate: Some(gate),
15692            after: io::Cursor::new(after.as_bytes().to_vec()),
15693        };
15694        let task = tokio::spawn(pump_stderr_to(
15695            reader,
15696            Arc::clone(ring),
15697            generation,
15698            DiscardSink,
15699        ));
15700        (StderrPump { task, generation }, release)
15701    }
15702
15703    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15704        for _ in 0..1000 {
15705            if done(&lock(ring)) {
15706                return;
15707            }
15708            tokio::time::sleep(Duration::from_millis(1)).await;
15709        }
15710        panic!(
15711            "ring never reached the expected state: {:?}",
15712            lock(ring).snapshot(None, None)
15713        );
15714    }
15715
15716    #[tokio::test(start_paused = true)]
15717    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15718        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15719        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15720
15721        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15722        let before_release = lock(&ring).snapshot(None, None);
15723        assert!(
15724            matches!(before_release.capture, CaptureState::Incomplete { .. }),
15725            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15726        );
15727
15728        // The restart: the next process starts and writes before the old
15729        // reader catches up.
15730        let next = lock(&ring).begin_process();
15731        lock(&ring).push_line_from(next, "next process booting");
15732        release.send(()).unwrap();
15733        wait_until(&ring, |ring| {
15734            ring.snapshot(None, None).capture == CaptureState::Captured
15735        })
15736        .await;
15737
15738        assert_eq!(
15739            untimed(lock(&ring).snapshot(None, None).entries),
15740            vec![
15741                line("booting"),
15742                line("config error: missing storage"),
15743                TailEntry::ProcessStart,
15744                line("next process booting"),
15745            ],
15746            "the crash's last line must survive a slow reader and stay in the crashed process's section"
15747        );
15748    }
15749
15750    #[tokio::test(start_paused = true)]
15751    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15752    ) {
15753        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15754        // `_held` is never fired: a descendant keeps the pipe open for the
15755        // whole test.
15756        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15757
15758        let started = Instant::now();
15759        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15760        assert_eq!(
15761            started.elapsed(),
15762            BOUND,
15763            "the restart must wait exactly the bound for a pipe that stays open, no longer"
15764        );
15765
15766        let next = lock(&ring).begin_process();
15767        lock(&ring).push_line_from(next, "next process booting");
15768        tokio::time::sleep(Duration::from_secs(60)).await;
15769
15770        let snapshot = lock(&ring).snapshot(None, None);
15771        match &snapshot.capture {
15772            CaptureState::Incomplete { reason } => assert!(
15773                reason.contains("had not reached EOF") && reason.contains("250ms"),
15774                "the reason must say what is missing and after how long: {reason}"
15775            ),
15776            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15777        }
15778        assert_eq!(
15779            untimed(snapshot.entries),
15780            vec![
15781                line("parent exiting"),
15782                TailEntry::ProcessStart,
15783                line("next process booting"),
15784            ]
15785        );
15786    }
15787
15788    #[tokio::test(start_paused = true)]
15789    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15790        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15791        let (pump, release) = held_pump(&ring, "one\n", "two\n");
15792        release.send(()).unwrap();
15793
15794        settle_stderr_pump("clean", &ring, pump, BOUND).await;
15795
15796        let snapshot = lock(&ring).snapshot(None, None);
15797        assert_eq!(snapshot.capture, CaptureState::Captured);
15798        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15799    }
15800
15801    #[tokio::test(start_paused = true)]
15802    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15803        const BURST: usize = 150;
15804        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15805        let path = capture.join("two-pipe.stderr.log");
15806        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15807        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15808        let generation = lock(&ring).begin_process();
15809        let stdout = (0..BURST)
15810            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15811            .collect::<String>();
15812        let stderr = (0..BURST)
15813            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15814            .collect::<String>();
15815        // Model a child that has written both bursts and exited. Hold the stdout
15816        // reader after one line and a partial prefix, as if pipe readiness or
15817        // capture-file writes had delayed it. The final line also needs EOF to
15818        // flush: neither complete nor partial buffered output may be discarded.
15819        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15820        let (release, gate) = oneshot::channel();
15821        let reader = HeldReader {
15822            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15823            gate: Some(gate),
15824            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15825        };
15826        let (finished, completion) = oneshot::channel();
15827        let stdout_sink = sink.clone();
15828        let pump = tokio::spawn(async move {
15829            pump_stdout_to(reader, stdout_sink).await;
15830            finished.send(()).unwrap();
15831        });
15832        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15833
15834        let started = Instant::now();
15835        settle_stdout_pump("two-pipe", pump, BOUND).await;
15836        assert_eq!(
15837            started.elapsed(),
15838            BOUND,
15839            "a late reader must not hold up restart"
15840        );
15841        assert_eq!(
15842            std::fs::read_to_string(&path).unwrap().lines().count(),
15843            BURST + 1
15844        );
15845
15846        release
15847            .send(())
15848            .expect("stdout reader must survive the exit drain bound");
15849        tokio::time::timeout(Duration::from_secs(1), completion)
15850            .await
15851            .expect("released stdout reader must reach EOF")
15852            .expect("stdout capture must finish at EOF, not be cancelled");
15853        let contents = std::fs::read_to_string(&path).unwrap();
15854        let mut lines = contents
15855            .lines()
15856            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15857            .collect::<Vec<_>>();
15858        let mut expected = (0..BURST)
15859            .flat_map(|i| {
15860                [
15861                    format!("out-{i:04}-{}", "o".repeat(64)),
15862                    format!("err-{i:04}-{}", "e".repeat(64)),
15863                ]
15864            })
15865            .collect::<Vec<_>>();
15866        lines.sort_unstable();
15867        expected.sort_unstable();
15868        assert_eq!(
15869            lines, expected,
15870            "both pipes must retain every complete line exactly once"
15871        );
15872    }
15873}
15874
15875/// Containment of a module's process tree (issue #109).
15876///
15877/// The behaviour these defend against is a module helper surviving its module:
15878/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
15879/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
15880/// compounds it.
15881///
15882/// They run against the SUPERVISOR rather than the job-object crate because the
15883/// claim is about teardown: a crate-level test proves a job can reap a tree, not
15884/// that the daemon's drain path reaches it.
15885///
15886/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
15887/// lane there is a separate containment path with its own tests.
15888#[cfg(all(test, windows))]
15889mod job_containment_tests {
15890    use super::*;
15891    use cortexkit_test_support::ScratchDir;
15892    use std::{
15893        path::{Path, PathBuf},
15894        sync::{Arc, Mutex},
15895        time::{Duration, Instant},
15896    };
15897
15898    /// The stub, expected beside this test executable.
15899    ///
15900    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
15901    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
15902    /// failure then reads as a broken test rather than an unbuilt dependency.
15903    fn stub_path() -> PathBuf {
15904        let mut path = std::env::current_exe().expect("current_exe available in tests");
15905        path.pop();
15906        path.pop();
15907        path.push("fake-aft-stub.exe");
15908        assert!(
15909            path.exists(),
15910            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15911             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15912            path.display()
15913        );
15914        path
15915    }
15916
15917    /// Poll for the grandchild pid the stub records, and parse it.
15918    fn read_grandchild_pid(path: &Path) -> u32 {
15919        let deadline = Instant::now() + Duration::from_secs(10);
15920        loop {
15921            if let Ok(contents) = std::fs::read_to_string(path) {
15922                if let Ok(pid) = contents.trim().parse() {
15923                    return pid;
15924                }
15925            }
15926            assert!(
15927                Instant::now() < deadline,
15928                "the stub never recorded a grandchild pid at {}",
15929                path.display()
15930            );
15931            std::thread::sleep(Duration::from_millis(10));
15932        }
15933    }
15934
15935    /// Everything one fixture run needs, so the two tests below differ in exactly
15936    /// one place: whether the child is contained.
15937    struct Fixture {
15938        _dir: ScratchDir,
15939        module_id: String,
15940        grandchild: u32,
15941        child: Option<SupervisedChild>,
15942        registry: Arc<Registry>,
15943        snapshot: Arc<Mutex<SupervisorSnapshot>>,
15944        terminal_ring: Arc<Mutex<TerminalRing>>,
15945        spawn_events: SpawnEventFeed,
15946    }
15947
15948    fn fixture(label: &str, module_id: &str) -> Fixture {
15949        let dir = ScratchDir::new(label);
15950        let pid_file = dir.join("grandchild.pid");
15951        let supervisor = Supervisor::new_for_test(
15952            Arc::new(Registry::default()),
15953            RestartPolicy::new(3, Duration::ZERO),
15954        );
15955        let runtime = supervisor.runtime_config();
15956        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15957        let spec = ModuleSpec {
15958            module_id: module_id.to_string(),
15959            program: stub_path(),
15960            // Zero args deliberately: a `--subc` argument would make the stub dial
15961            // a daemon that is not there, and the failure would land in the same
15962            // stderr ring this fixture exists to keep quiet.
15963            args: Vec::new(),
15964            env: vec![
15965                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15966                (
15967                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15968                    pid_file.display().to_string(),
15969                ),
15970            ],
15971            reserved: false,
15972            reserved_prefixes: Vec::new(),
15973            protocol: ModuleProtocol::Subc,
15974            overlap: Default::default(),
15975        };
15976        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15977            .expect("spawn the supervised fixture");
15978        let grandchild = read_grandchild_pid(&pid_file);
15979        Fixture {
15980            _dir: dir,
15981            module_id: module_id.to_string(),
15982            grandchild,
15983            child: Some(child),
15984            registry: Arc::new(Registry::default()),
15985            snapshot,
15986            terminal_ring: Arc::clone(&runtime.terminal_ring),
15987            spawn_events: SpawnEventFeed::default(),
15988        }
15989    }
15990
15991    impl Fixture {
15992        /// Drain through the supervisor's own teardown path.
15993        async fn drain(&mut self) {
15994            let child = self
15995                .child
15996                .take()
15997                .expect("the fixture child is still present");
15998            drain_child_to_state(
15999                &self.module_id,
16000                ModuleProtocol::Subc,
16001                // No forwarding table in this fixture, so nothing reaches the
16002                // child over a connection.
16003                StopNotice::NotSent,
16004                &self.registry,
16005                None,
16006                &self.snapshot,
16007                &self.terminal_ring,
16008                &self.spawn_events,
16009                child,
16010                Duration::from_millis(500),
16011                ModuleState::Stopped,
16012                Some(false),
16013            )
16014            .await
16015            .expect("drain the supervised fixture");
16016        }
16017    }
16018
16019    /// Teardown reaps the grandchild, not merely the direct child.
16020    ///
16021    /// This is the assertion the change exists for. Before containment the
16022    /// grandchild survived: it is a separate process, and `start_kill` is
16023    /// `TerminateProcess` scoped to one pid.
16024    #[tokio::test]
16025    async fn teardown_reaps_the_grandchild() {
16026        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16027        let grandchild = fixture.grandchild;
16028
16029        assert!(
16030            subc_jobobject::process_exists(grandchild),
16031            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16032        );
16033
16034        fixture.drain().await;
16035
16036        assert!(
16037            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16038            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16039        );
16040    }
16041
16042    /// The mutation control: with containment withheld, the grandchild survives
16043    /// the same kill.
16044    ///
16045    /// This is the defect reproduction from #109 — a direct-child kill reaches
16046    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16047    /// supervisor because `spawn_and_mark_running` now always contains on
16048    /// Windows, which is the point: there is no longer a path that spawns
16049    /// uncontained, so the control has to construct one.
16050    ///
16051    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16052    /// grandchild ever dies here, that test is passing for a reason unrelated to
16053    /// the job object and the containment claim is unproven.
16054    #[test]
16055    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16056        let dir = ScratchDir::new("teardown-uncontained");
16057        let pid_file = dir.join("grandchild.pid");
16058        let mut child = std::process::Command::new(stub_path())
16059            .env("FAKE_AFT_NEVER_CONNECT", "1")
16060            .env(
16061                "FAKE_AFT_GRANDCHILD_PID_FILE",
16062                pid_file.display().to_string(),
16063            )
16064            .stdin(std::process::Stdio::null())
16065            .stdout(std::process::Stdio::null())
16066            .stderr(std::process::Stdio::null())
16067            .spawn()
16068            .expect("spawn the uncontained fixture");
16069        let grandchild = read_grandchild_pid(&pid_file);
16070
16071        // Exactly what the pre-fix teardown did: kill the direct child.
16072        child.kill().expect("kill the direct child");
16073        let _ = child.wait();
16074
16075        assert!(
16076            subc_jobobject::process_exists(grandchild),
16077            "grandchild {grandchild} died with the direct child, so this control no longer \
16078             distinguishes contained from uncontained teardown and the regression test is \
16079             passing vacuously"
16080        );
16081
16082        // The orphan this control demonstrates is the leak the fix prevents, so
16083        // the control must not leave one behind.
16084        kill_tree(grandchild);
16085    }
16086
16087    /// Crash durability: closing the containment handle reaps the tree with no
16088    /// teardown code running at all.
16089    ///
16090    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16091    /// call anything — and it is why containment is a kernel property of the
16092    /// handle rather than a step in the drain. Discovered by getting the
16093    /// mutation control wrong: clearing `job` to "disable" containment instead
16094    /// killed the tree, which is the guarantee, not a mistake.
16095    #[tokio::test]
16096    async fn dropping_containment_reaps_the_grandchild() {
16097        let mut fixture = fixture("drop-containment", "tree-drop");
16098        let grandchild = fixture.grandchild;
16099
16100        assert!(subc_jobobject::process_exists(grandchild));
16101
16102        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16103        fixture.child.as_mut().expect("child present").job = None;
16104
16105        assert!(
16106            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16107            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16108             crash would leave the tree behind"
16109        );
16110    }
16111
16112    /// Kill a pid and its tree, then confirm it is gone.
16113    fn kill_tree(pid: u32) {
16114        let _ = std::process::Command::new("taskkill.exe")
16115            .args(["/PID", &pid.to_string(), "/T", "/F"])
16116            .stdin(std::process::Stdio::null())
16117            .stdout(std::process::Stdio::null())
16118            .stderr(std::process::Stdio::null())
16119            .status();
16120        assert!(
16121            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16122            "could not clean up grandchild {pid}"
16123        );
16124    }
16125}
16126
16127#[cfg(test)]
16128mod privacy_trampoline_configuration_tests {
16129    #[tokio::test]
16130    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16131    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16132        #[cfg(target_os = "macos")]
16133        {
16134            let supervisor = super::Supervisor::new(
16135                std::sync::Arc::new(crate::Registry::default()),
16136                super::RestartPolicy::default(),
16137            );
16138            let error = supervisor.spawn(spec()).unwrap_err();
16139            assert!(
16140                error
16141                    .to_string()
16142                    .contains("no privacy trampoline configured"),
16143                "{error}"
16144            );
16145        }
16146    }
16147
16148    #[tokio::test]
16149    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16150    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16151        #[cfg(target_os = "macos")]
16152        {
16153            let supervisor = super::Supervisor::new(
16154                std::sync::Arc::new(crate::Registry::default()),
16155                super::RestartPolicy::default(),
16156            )
16157            .with_privacy_trampoline(std::env::current_exe().unwrap());
16158            let error = supervisor.spawn(spec()).unwrap_err();
16159            assert!(
16160                error
16161                    .to_string()
16162                    .contains("binary does not implement the privacy trampoline protocol"),
16163                "{error}"
16164            );
16165        }
16166    }
16167
16168    #[cfg(target_os = "macos")]
16169    fn spec() -> super::ModuleSpec {
16170        super::ModuleSpec {
16171            module_id: "privacy-configuration".into(),
16172            program: "/bin/sleep".into(),
16173            args: vec!["30".into()],
16174            env: vec![],
16175            reserved: false,
16176            reserved_prefixes: vec![],
16177            protocol: subc_control::ModuleProtocol::None,
16178            overlap: super::ModuleOverlap::Exclusive,
16179        }
16180    }
16181}
16182
16183#[cfg(test)]
16184mod privacy_exec_boundary_tests {
16185    #[cfg(target_os = "macos")]
16186    use super::*;
16187    #[cfg(target_os = "macos")]
16188    use std::{
16189        io::{Read, Write},
16190        net::{TcpListener, TcpStream},
16191    };
16192
16193    /// Unit-test-only pause at the actual early image read, not at a later
16194    /// status read. Production supervisors never inspect this environment key.
16195    #[cfg(target_os = "macos")]
16196    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16197        if let Some((_, path)) = spec
16198            .env
16199            .iter()
16200            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16201        {
16202            let mut barrier = TcpStream::connect(path).unwrap();
16203            barrier
16204                .set_read_timeout(Some(Duration::from_secs(30)))
16205                .unwrap();
16206            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16207            let mut release = [0];
16208            barrier.read_exact(&mut release).unwrap();
16209            assert_eq!(&release, b"X");
16210        }
16211    }
16212
16213    #[cfg(target_os = "macos")]
16214    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16215        ModuleSpec {
16216            module_id: "privacy-boundary".into(),
16217            program: program.into(),
16218            args: args.iter().map(|arg| (*arg).into()).collect(),
16219            env: vec![],
16220            reserved: false,
16221            reserved_prefixes: vec![],
16222            protocol: ModuleProtocol::None,
16223            overlap: ModuleOverlap::Exclusive,
16224        }
16225    }
16226
16227    #[cfg(target_os = "macos")]
16228    async fn accept(listener: TcpListener) -> TcpStream {
16229        // Socket readiness, not elapsed time, establishes both pause points.
16230        let listener = tokio::net::TcpListener::from_std({
16231            listener.set_nonblocking(true).unwrap();
16232            listener
16233        })
16234        .unwrap();
16235        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16236            .await
16237            .unwrap()
16238            .unwrap();
16239        let stream = stream.into_std().unwrap();
16240        stream.set_nonblocking(false).unwrap();
16241        stream
16242            .set_read_timeout(Some(Duration::from_secs(30)))
16243            .unwrap();
16244        stream
16245    }
16246
16247    #[tokio::test]
16248    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16249    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16250        #[cfg(target_os = "macos")]
16251        {
16252            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16253            // Loopback sockets also work when the replay adapter's TMPDIR is
16254            // longer than Darwin's Unix-domain socket path limit.
16255            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16256            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16257            let record = root.join("live-children.json");
16258            let supervisor =
16259                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16260                    .with_live_children_record(&record);
16261            let runtime = supervisor.runtime_config();
16262            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16263            let mut spec = spec("/bin/sleep", &["30"]);
16264            spec.env = vec![
16265                (
16266                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16267                    exec_listener.local_addr().unwrap().to_string(),
16268                ),
16269                (
16270                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16271                    sample_listener.local_addr().unwrap().to_string(),
16272                ),
16273            ];
16274            let spawn = tokio::task::spawn_blocking(move || {
16275                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16276            });
16277            let mut sample = accept(sample_listener).await;
16278            let mut pid = [0; 4];
16279            sample.read_exact(&mut pid).unwrap();
16280            let pid = u32::from_ne_bytes(pid);
16281            let mut exec = accept(exec_listener).await;
16282            let mut ready = [0];
16283            exec.read_exact(&mut ready).unwrap();
16284            assert_eq!(&ready, b"R");
16285            // The early read is guaranteed to see a real, non-null trampoline
16286            // image: the fixture has reached its barrier and cannot exec yet.
16287            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16288            assert_eq!(
16289                observe_spawned_image(pid).unwrap().executable,
16290                Some(trampoline)
16291            );
16292            sample.write_all(b"X").unwrap();
16293            let mut child = spawn.await.unwrap();
16294            let early = crate::live_children::read_record(&record).unwrap();
16295            assert_eq!(early.len(), 1);
16296            assert_eq!(early[0].pid, pid);
16297            assert_eq!(
16298                early[0].executable, None,
16299                "unconfirmed trampoline image entered the roster"
16300            );
16301            assert!(child.report_ready.get().is_none());
16302            // The barrier's duration is unrelated to the production five-second
16303            // exec budget. Start the test's confirmation budget upon release.
16304            child.privacy_exec.as_mut().unwrap().deadline =
16305                tokio::time::Instant::now() + Duration::from_secs(30);
16306            exec.write_all(b"X").unwrap();
16307            child.confirm_privacy_exec().await;
16308            assert_eq!(child.spawn_failure, None);
16309            assert!(child.report_ready.get().is_some());
16310            let confirmed = crate::live_children::read_record(&record).unwrap();
16311            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16312            assert_ne!(module, trampoline);
16313            assert_eq!(confirmed[0].executable, Some(module.into()));
16314            child.start_kill().unwrap();
16315            child.wait().await.unwrap();
16316            child.release_roster();
16317        }
16318    }
16319
16320    #[tokio::test]
16321    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16322    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16323        #[cfg(target_os = "macos")]
16324        {
16325            let registry = Arc::new(Registry::default());
16326            let policy = RestartPolicy::new(0, Duration::ZERO);
16327            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16328            let runtime = supervisor.runtime_config();
16329            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16330            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16331            // Drive spawn and confirmation separately instead of starting the
16332            // monitor. WNOWAIT observes a real exit without consuming its status,
16333            // so confirmation's first try_wait must take the already-exited arm.
16334            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16335            let pid = child.pid;
16336            tokio::task::spawn_blocking(move || {
16337                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16338            })
16339            .await
16340            .unwrap()
16341            .unwrap();
16342            child.privacy_exec.as_mut().unwrap().deadline =
16343                tokio::time::Instant::now() + Duration::from_secs(30);
16344            let status = child.wait().await.unwrap();
16345            assert_eq!(status.code(), Some(121));
16346            assert!(child.privacy_exec.is_none());
16347            assert!(
16348                child.report_ready.get().is_none(),
16349                "an exited module must not publish a live pid"
16350            );
16351            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16352            on_child_exit(
16353                &spec,
16354                policy,
16355                &registry,
16356                &snapshot,
16357                &runtime.terminal_ring,
16358                &runtime.spawn_events,
16359                &runtime.child_roster,
16360                report,
16361            )
16362            .await;
16363            let state = lock_snapshot(&snapshot).unwrap();
16364            assert_eq!(state.state, ModuleState::Failed);
16365            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16366            assert_eq!(state.reported_pid(), None);
16367            drop(state);
16368            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16369            assert_eq!(history.entries.len(), 1);
16370            let terminal = &history.entries[0];
16371            assert_eq!(terminal.exit_code, Some(121));
16372            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16373            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16374            assert_eq!(
16375                terminal.disposition_detail.as_deref(),
16376                Some(policy.budget_exhausted_detail().as_str()),
16377                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16378            );
16379            assert_eq!(child.spawn_failure, None);
16380            child.release_roster();
16381        }
16382    }
16383}
16384
16385/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16386/// descriptor 3, without an environment copy. The shell records the nonce
16387/// and its environment after exec so these tests observe the real handover.
16388#[cfg(all(test, unix))]
16389mod launch_nonce_descriptor_tests {
16390    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16391    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16392    use cortexkit_test_support::ScratchDir;
16393    use std::{
16394        path::PathBuf,
16395        sync::{Arc, Mutex},
16396        time::{Duration, Instant},
16397    };
16398
16399    async fn probe(role: super::SpawnRole) {
16400        let scratch = ScratchDir::new("launch-nonce-descriptor");
16401        let fd_copy = scratch.join("from-descriptor");
16402        let env_copy = scratch.join("environment");
16403        let script = format!(
16404            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16405            fd = fd_copy.display(), env = env_copy.display(),
16406        );
16407        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16408        let spec = ModuleSpec {
16409            module_id: "nonce-descriptor-probe".to_string(),
16410            program: PathBuf::from("/bin/sh"),
16411            args: vec!["-c".to_string(), script],
16412            env: vec![
16413                xdg("XDG_DATA_HOME"),
16414                xdg("XDG_RUNTIME_DIR"),
16415                xdg("XDG_CONFIG_HOME"),
16416                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16417            ],
16418            reserved: true,
16419            reserved_prefixes: Vec::new(),
16420            protocol: ModuleProtocol::Subc,
16421            overlap: Default::default(),
16422        };
16423        let handle = SupervisorHandle::new();
16424        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16425        let roster = ChildRoster::default();
16426        #[cfg(target_os = "macos")]
16427        {
16428            let path = super::test_privacy_trampoline();
16429            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16430        }
16431        let child = super::spawn_child_in_slot(
16432            &spec,
16433            None,
16434            Some(&handle),
16435            &ring,
16436            None,
16437            &roster,
16438            #[cfg(target_os = "linux")]
16439            None,
16440            role,
16441            matches!(role, super::SpawnRole::SwapCandidate),
16442        )
16443        .expect("spawn probe");
16444        let deadline = Instant::now() + Duration::from_secs(10);
16445        while !(fd_copy.exists() && env_copy.exists()) {
16446            assert!(Instant::now() < deadline, "probe never wrote its copies");
16447            tokio::time::sleep(Duration::from_millis(20)).await;
16448        }
16449        let nonce = std::fs::read_to_string(fd_copy).unwrap();
16450        assert!(!nonce.is_empty());
16451        let environment = std::fs::read_to_string(env_copy).unwrap();
16452        assert!(environment
16453            .lines()
16454            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16455        let copy = environment
16456            .lines()
16457            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16458        assert_eq!(
16459            copy, None,
16460            "Unix children must never receive the environment nonce"
16461        );
16462        if matches!(role, super::SpawnRole::Plain) {
16463            assert_eq!(
16464                handle.spawn_nonce(&spec.module_id).as_deref(),
16465                Some(nonce.as_str())
16466            );
16467        }
16468        drop(child);
16469    }
16470
16471    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16472    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16473        probe(super::SpawnRole::Plain).await;
16474    }
16475
16476    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16477    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16478        probe(super::SpawnRole::SwapCandidate).await;
16479    }
16480}
16481
16482#[cfg(all(test, target_os = "linux"))]
16483mod cgroup_containment_tests {
16484    use super::*;
16485    use cortexkit_test_support::ScratchDir;
16486
16487    fn running(pid: u32) -> bool {
16488        // An orphan can remain a zombie until the container init reaps it.
16489        std::fs::read_to_string(format!("/proc/{pid}/stat"))
16490            .ok()
16491            .and_then(|stat| {
16492                stat.rsplit_once(") ")
16493                    .map(|(_, rest)| rest.starts_with('Z'))
16494            })
16495            .is_some_and(|zombie| !zombie)
16496    }
16497
16498    #[tokio::test]
16499    async fn linux_teardown_reaps_the_grandchild() {
16500        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16501    }
16502
16503    #[tokio::test]
16504    async fn linux_shutdown_straggler_reaps_the_grandchild() {
16505        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16506    }
16507
16508    async fn teardown_tree(test_name: &str, shutdown: bool) {
16509        let dir = ScratchDir::new(test_name);
16510        let root = PathBuf::from(format!(
16511            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16512            std::process::id(),
16513            unix_ms_now()
16514        ));
16515        if let Err(error) = std::fs::create_dir(&root) {
16516            assert!(
16517                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16518                "required cgroup test cannot execute: {error}"
16519            );
16520            eprintln!(
16521                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16522                root.display()
16523            );
16524            return;
16525        }
16526        let placement = subc_cgroup::prepare_at(&root)
16527            .expect("prepare isolated kernel cgroup")
16528            .expect("isolated cgroup is delegated");
16529        let module_id = "tree-teardown";
16530        let module = placement
16531            .module_path(module_id)
16532            .expect("create isolated module cgroup");
16533        if !module.join("cgroup.kill").exists() {
16534            std::fs::remove_dir(&module).unwrap();
16535            std::fs::remove_dir(root.join("subc-modules")).unwrap();
16536            std::fs::remove_dir(&root).unwrap();
16537            assert!(
16538                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16539                "required cgroup.kill interface unavailable"
16540            );
16541            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16542            return;
16543        }
16544        let supervisor = Supervisor::new_for_test(
16545            Arc::new(Registry::default()),
16546            RestartPolicy::new(3, Duration::ZERO),
16547        )
16548        .with_cgroup_placement(Some(placement));
16549        let mut runtime = supervisor.runtime_config();
16550        runtime.child_roster = runtime
16551            .child_roster
16552            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16553        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16554        let pid_file = dir.join("grandchild.pid");
16555        let spec = ModuleSpec {
16556            module_id: module_id.to_string(),
16557            program: PathBuf::from("/bin/sh"),
16558            args: vec![
16559                "-c".into(),
16560                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16561                "fixture".into(),
16562                pid_file.display().to_string(),
16563            ],
16564            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16565                .into_iter()
16566                .map(|key| (key.to_string(), dir.display().to_string()))
16567                .collect(),
16568            reserved: false,
16569            reserved_prefixes: Vec::new(),
16570            protocol: ModuleProtocol::None,
16571            overlap: Default::default(),
16572        };
16573        let child =
16574            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16575        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16576        let grandchild: u32 = loop {
16577            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16578                if let Ok(pid) = contents.trim().parse() {
16579                    break pid;
16580                }
16581            }
16582            assert!(
16583                tokio::time::Instant::now() < deadline,
16584                "grandchild pid was not recorded"
16585            );
16586            tokio::time::sleep(Duration::from_millis(10)).await;
16587        };
16588        assert!(
16589            running(grandchild),
16590            "grandchild must be alive before teardown"
16591        );
16592        if shutdown {
16593            let mut child = child;
16594            crate::child_roster::end_children_for_daemon_shutdown(
16595                &runtime.child_roster,
16596                false,
16597                std::future::pending(),
16598            )
16599            .await;
16600            child.wait().await.expect("reap shutdown straggler");
16601        } else {
16602            drain_child_to_state(
16603                module_id,
16604                ModuleProtocol::None,
16605                StopNotice::NotSent,
16606                &Registry::default(),
16607                None,
16608                &snapshot,
16609                &runtime.terminal_ring,
16610                &SpawnEventFeed::default(),
16611                child,
16612                Duration::from_millis(100),
16613                ModuleState::Stopped,
16614                Some(false),
16615            )
16616            .await
16617            .expect("real supervisor teardown");
16618        }
16619        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16620        while running(grandchild) && tokio::time::Instant::now() < deadline {
16621            tokio::time::sleep(Duration::from_millis(10)).await;
16622        }
16623        let survived = running(grandchild);
16624        // Kill a surviving grandchild so a failed test does not leave it behind.
16625        if survived {
16626            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16627            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16628            tokio::time::sleep(Duration::from_millis(100)).await;
16629        }
16630        if module.exists() {
16631            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16632        }
16633        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16634        std::fs::remove_dir(&root).unwrap();
16635        assert!(
16636            !survived,
16637            "grandchild {grandchild} outlived module teardown"
16638        );
16639        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16640    }
16641}