Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    /// Conservative monotonic bound sampled before `command.spawn()`, not when
362    /// identity recording finishes or the spawn generation is published.
363    spawned_at: Instant,
364    spawned_at_ms: u64,
365    spawned_from: PathBuf,
366    launch_spec_digest: LaunchSpecDigest,
367    spawned_file_identity: Option<SpawnedFileIdentity>,
368    process_start_time: Option<u64>,
369    process_identity: Option<ProcessIdentity>,
370    pid: u32,
371    /// This process's entry in the daemon's child roster, released when the
372    /// process is reaped or this handle is dropped.
373    roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377    fn id(&self) -> Option<u32> {
378        Some(self.pid)
379    }
380
381    fn process_identity(&self) -> Option<ProcessIdentity> {
382        self.process_identity
383    }
384
385    async fn wait(&mut self) -> io::Result<ExitStatus> {
386        #[cfg(target_os = "macos")]
387        self.confirm_privacy_exec().await;
388        // The roster entry is NOT released here. A daemon shutdown waits for the
389        // roster to empty and then exits the process, so releasing at the reap
390        // let it exit before the exit handler wrote this child's terminal record
391        // (the stderr drain and snapshot update sit in between), and the
392        // shutdown's own `daemon_shutdown` record was intermittently lost. The
393        // caller releases it after recording the exit (`release_roster`), and
394        // dropping the handle releases it too.
395        let result = self.child.wait().await;
396        #[cfg(target_os = "linux")]
397        if result.is_ok() {
398            if let Some(placement) = self.cgroup_placement.as_ref() {
399                cleanup_reaped_cgroup(placement, &self.module_id).await;
400                // Keep ownership while awaiting kernel population changes: a
401                // drain timeout may cancel this wait and then escalate/reap.
402                self.cgroup_placement = None;
403            }
404        }
405        result
406    }
407
408    #[cfg(target_os = "macos")]
409    async fn confirm_privacy_exec(&mut self) {
410        let Some(pending) = &mut self.privacy_exec else {
411            return;
412        };
413        let result = tokio::time::timeout_at(pending.deadline, async {
414            let mut record = Vec::new();
415            loop {
416                let mut ready = pending.reader.readable().await?;
417                let read = ready.try_io(|reader| {
418                    use std::io::Read;
419                    let mut reader = reader.get_ref();
420                    let mut buffer = [0u8; 256];
421                    reader.read(&mut buffer).map(|count| (count, buffer))
422                });
423                match read {
424                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425                    Ok(Ok((count, buffer))) => {
426                        if record.len() + count > 1024 {
427                            return Err(io::Error::other(
428                                "privacy exec refusal record is too long",
429                            ));
430                        }
431                        record.extend_from_slice(&buffer[..count]);
432                    }
433                    Ok(Err(error)) => return Err(error),
434                    Err(_) => continue,
435                }
436            }
437        })
438        .await;
439        #[cfg(test)]
440        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441            use tokio::io::{AsyncReadExt, AsyncWriteExt};
442            if stream.is_none() {
443                *stream = Some(
444                    tokio::net::TcpStream::connect(address.as_str())
445                        .await
446                        .unwrap(),
447                );
448                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449                stream
450                    .as_mut()
451                    .unwrap()
452                    .write_all(&self.pid.to_ne_bytes())
453                    .await
454                    .unwrap();
455            }
456            let mut reply = [0];
457            stream
458                .as_mut()
459                .unwrap()
460                .read_exact(&mut reply)
461                .await
462                .unwrap();
463            pending.confirmation_pause = None;
464            if &reply == b"F" {
465                Ok(Err(io::Error::other("injected late exec refusal")))
466            } else {
467                assert_eq!(&reply, b"X");
468                result
469            }
470        } else {
471            result
472        };
473        // Keep the reader in self across await: select cancellation must not
474        // discard the handshake or reset its original five-second deadline.
475        let pending = self.privacy_exec.as_ref().expect("pending exec");
476        let cause = match result {
477            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478            Ok(Err(error)) => Some(format!(
479                "privacy identity exec acknowledgement failed: {error}"
480            )),
481            Ok(Ok(record)) if !record.is_empty() => Some(
482                std::str::from_utf8(&record)
483                    .ok()
484                    .and_then(|record| {
485                        record
486                            .trim()
487                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488                    })
489                    .filter(|cause| !cause.is_empty())
490                    .unwrap_or("invalid privacy exec refusal record")
491                    .to_string(),
492            ),
493            Ok(Ok(_)) => match self.child.try_wait() {
494                // Empty EOF is the exec acknowledgement. A real module may exit
495                // immediately, including with a reserved trampoline status; no
496                // image is admitted, and its ordinary exit contract stays intact.
497                Ok(Some(_status)) => None,
498                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499                Ok(None) => {
500                    let image = observe_spawned_image(self.pid);
501                    if let Some(image) = image.filter(|image| {
502                        image.executable.is_some()
503                            && image.executable != pending.trampoline
504                            && (image.executable == pending.expected || pending.script)
505                    }) {
506                        if let Err(error) = self
507                            .roster_guard
508                            .as_ref()
509                            .map(|guard| guard.confirm_executable(image))
510                            .transpose()
511                        {
512                            Some(format!(
513                                "could not durably record confirmed module image: {error}"
514                            ))
515                        } else {
516                            let _ = self.report_ready.set(());
517                            info!(module_id = %pending.module_id, pid = self.pid,
518                                "module spawned with own privacy identity (responsibility disclaimed)");
519                            None
520                        }
521                    } else if image.is_none()
522                        || image.is_some_and(|image| image.executable.is_none())
523                    {
524                        // A process can exit between try_wait and the kernel
525                        // image read. Empty EOF already acknowledged exec, so
526                        // preserve that module's ordinary exit rather than
527                        // mislabel a disappearing image as trampoline refusal.
528                        // Keep pending in self across await so cancellation does
529                        // not discard validation or reset its original deadline.
530                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531                            Ok(Ok(_status)) => None,
532                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534                        }
535                    } else {
536                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537                    }
538                }
539            },
540        };
541        let pending = self.privacy_exec.take().expect("pending exec");
542        // Exec confirmation describes launch success, not roster retirement.
543        // Keep even a reaped child recorded until its terminal history exists.
544        self.exec.0.finish(if cause.is_some() {
545            crate::activity::ExecState::Refused
546        } else {
547            crate::activity::ExecState::CommittedExec
548        });
549        if let Some(cause) = cause {
550            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551            self.spawn_failure = Some(cause);
552            // No image is admitted on failure. Reach the entire fresh process
553            // group, including a module which spawned a helper before refusal.
554            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556            }
557            let _ = self.child.start_kill();
558        }
559    }
560
561    /// Releases this child's daemon-shutdown roster entry once its exit has
562    /// been recorded. The pid is already reaped and free for reuse, so the
563    /// entry must not outlive the record any longer than that.
564    fn release_roster(&mut self) {
565        self.roster_guard = None;
566    }
567
568    /// Kill the child and, where containment is available, its process tree.
569    ///
570    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
571    /// helper process leaked the helper — the Synapse embedding module's CUDA
572    /// worker holds the GPU allocation, so the leak cost VRAM until the next
573    /// restart of something else. Terminating the job reaches grandchildren that
574    /// a tree walk cannot, including one whose parent has already exited and
575    /// been reparented away.
576    ///
577    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
578    /// direct-child kill still decides the outcome, so containment can never
579    /// change whether a module is reported as stopped.
580    fn start_kill(&mut self) -> io::Result<()> {
581        #[cfg(windows)]
582        if let Some(job) = &self.job {
583            if let Err(error) = job.terminate() {
584                debug!(
585                    error = %error,
586                    "job termination failed; the direct-child kill still owns the outcome"
587                );
588            }
589        }
590        #[cfg(target_os = "linux")]
591        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592        self.child.start_kill()
593    }
594
595    async fn drain_stderr(&mut self, module_id: &str) {
596        if let Some(pump) = self.stdout_pump.take() {
597            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598        }
599
600        let Some(pump) = self.stderr_pump.take() else {
601            return;
602        };
603        settle_stderr_pump(
604            module_id,
605            &self.stderr_ring,
606            pump,
607            STDERR_PUMP_DRAIN_TIMEOUT,
608        )
609        .await;
610    }
611}
612
613/// Bound the restart's wait, not the lifetime of the exited child's stdout
614/// reader. Slow pipe reads or capture writes must not turn the bound into lost
615/// output; dropping the handle lets the reader finish at EOF, like stderr.
616async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617    match timeout(bound, &mut pump).await {
618        Ok(Ok(())) => {}
619        Ok(Err(error)) => {
620            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621        }
622        Err(_) => {
623            drop(pump);
624            warn!(
625                module_id,
626                waited = ?bound,
627                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628            );
629        }
630    }
631}
632
633/// The reader task for one process's stderr, with the ring generation its
634/// lines are attributed to.
635struct StderrPump {
636    task: JoinHandle<()>,
637    generation: u64,
638}
639
640/// Retire an exited process's stderr reader and wait up to `bound` for it to
641/// reach EOF. A reader still running at the bound is detached, not stopped: it
642/// keeps filling the exited process's section of the ring until its pipe
643/// closes, and the tail reads `Incomplete` until then. See
644/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
645async fn settle_stderr_pump(
646    module_id: &str,
647    ring: &Arc<Mutex<StderrRing>>,
648    pump: StderrPump,
649    bound: Duration,
650) {
651    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652    let StderrPump {
653        mut task,
654        generation,
655    } = pump;
656    lock().retire_pump(generation);
657    match timeout(bound, &mut task).await {
658        Ok(Ok(())) => {}
659        Ok(Err(err)) => {
660            let mut ring = lock();
661            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662            ring.finish_pump(generation);
663            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664        }
665        Err(_) => {
666            // Dropping the handle detaches the task; it ends at EOF on its pipe.
667            drop(task);
668            lock().mark_pump_late(
669                generation,
670                format!(
671                    "stderr of the exited process had not reached EOF {bound:?} after it was \
672                     retired (a descendant may still hold the pipe open); lines it still \
673                     writes are kept in that process's section"
674                ),
675            );
676            warn!(
677                module_id,
678                waited = ?bound,
679                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680            );
681        }
682    }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687    EVENTS.get_or_init(|| {
688        let (sender, _receiver) = watch::channel(0);
689        sender
690    })
691}
692
693pub(crate) fn notify_registration_release() {
694    let events = registration_release_events();
695    let next_generation = (*events.borrow()).wrapping_add(1);
696    events.send_replace(next_generation);
697}
698
699/// How to launch one singleton module process.
700#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702    pub module_id: String,
703    pub program: PathBuf,
704    pub args: Vec<String>,
705    pub env: Vec<(String, String)>,
706    /// When true this is a reserved module: each spawn gets a fresh one-time launch
707    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
708    /// process can register this module_id (a security-boundary module like the
709    /// credential vault must not be impersonable while it is down/restarting).
710    pub reserved: bool,
711    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
712    /// Prefixes come from daemon config and must end in `:` before they reach the
713    /// supervisor; the owner module's current spawn nonce authorizes claims under
714    /// each prefix.
715    pub reserved_prefixes: Vec<String>,
716    /// The wire protocol this module speaks, as DECLARED in daemon config.
717    ///
718    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
719    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
720    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
721    /// and NO launch nonce, and a clean exit the daemon did not request is
722    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
723    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
724    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
725    /// because a process ignores an environment variable it does not read.
726    ///
727    /// The argument is the part that cannot be "harmless to a process that
728    /// ignores it": a stock binary exits on an unknown flag before it listens
729    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
730    /// first conformance run against this mode found it. The nonce is withheld
731    /// because a process that will never present it gains nothing from holding
732    /// it, and a secret in the environment of a process that does not need it is
733    /// a leak surface for no benefit.
734    pub protocol: ModuleProtocol,
735    /// Whether two processes of this module may run at once, which is what a
736    /// blue/green swap does for the length of its overlap. Declared in daemon
737    /// config because the daemon must be able to answer it while the module is
738    /// down, and so a module cannot talk itself into it after registering.
739    pub overlap: ModuleOverlap,
740}
741
742/// Only the configured launch inputs enter this value; spawned connection paths,
743/// nonces and inherited environment never do. Its bytes remain daemon-private.
744#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749        f.write_str("LaunchSpecDigest([redacted])")
750    }
751}
752
753impl LaunchSpecDigest {
754    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755        fn field(hash: &mut Sha256, bytes: &[u8]) {
756            hash.update((bytes.len() as u64).to_le_bytes());
757            hash.update(bytes);
758        }
759        let mut hash = Sha256::new();
760        if let Some(program) = spec.program.to_str() {
761            field(&mut hash, b"utf8-program");
762            field(&mut hash, program.as_bytes());
763        } else {
764            // The daemon config is UTF-8, but in-process callers may provide
765            // native paths: lossy conversion would collapse distinct programs.
766            #[cfg(unix)]
767            {
768                use std::os::unix::ffi::OsStrExt;
769                field(&mut hash, b"unix-program");
770                field(&mut hash, spec.program.as_os_str().as_bytes());
771            }
772            #[cfg(windows)]
773            {
774                use std::os::windows::ffi::OsStrExt;
775                field(&mut hash, b"windows-program");
776                let bytes = spec
777                    .program
778                    .as_os_str()
779                    .encode_wide()
780                    .flat_map(u16::to_le_bytes)
781                    .collect::<Vec<_>>();
782                field(&mut hash, &bytes);
783            }
784            #[cfg(not(any(unix, windows)))]
785            field(&mut hash, spec.program.to_string_lossy().as_bytes());
786        }
787        hash.update((spec.args.len() as u64).to_le_bytes());
788        for arg in &spec.args {
789            field(&mut hash, arg.as_bytes());
790        }
791        let mut env = spec.env.iter().collect::<Vec<_>>();
792        env.sort_unstable();
793        hash.update((env.len() as u64).to_le_bytes());
794        for (key, value) in env {
795            field(&mut hash, key.as_bytes());
796            field(&mut hash, value.as_bytes());
797        }
798        field(
799            &mut hash,
800            match spec.protocol {
801                ModuleProtocol::Subc => b"subc",
802                ModuleProtocol::None => b"none",
803            },
804        );
805        Self(hash.finalize().into())
806    }
807}
808
809/// Whether a module tolerates a second process of itself running alongside.
810///
811/// Most modules are single-writer on their store (a WAL, a capture log, a
812/// resident index behind a writer barrier), and two processes on one store
813/// corrupt it. So a swap, which overlaps the old and new process by design,
814/// is refused unless the module's config opts in with `overlap: "safe"`.
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817    /// Never run two processes of this module at once. The default.
818    #[default]
819    Exclusive,
820    /// The module has said a second process of itself is harmless for the
821    /// length of a swap.
822    ///
823    /// Declare it only if a second instance can run for a few seconds without
824    /// touching ANY single-writer store: every database, WAL, index, projector
825    /// and scheduled job the module owns. A lease on part of that state is not
826    /// enough. broca's session lease guards WAL appends while its run index, its
827    /// store projector and its archive fold timer (which unlinks live WAL files)
828    /// stay single-writer, so broca is exclusive despite holding a lease. The
829    /// refusal only fires after this has been decided, so the decision is the
830    /// check.
831    Safe,
832}
833
834impl ModuleOverlap {
835    pub fn as_str(self) -> &'static str {
836        match self {
837            Self::Exclusive => "exclusive",
838            Self::Safe => "safe",
839        }
840    }
841}
842
843/// Environment variable telling a spawned module which case it was started
844/// for, before it sends HELLO. Only a swap candidate carries it, as
845/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
846///
847/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
848/// longer because nobody waits on it, while a plain restart must flip ready
849/// quickly because callers see `module_warming` until it does. Absence means
850/// plain restart, the safe reading. The daemon trusts nothing about it; the
851/// candidate is proven by its launch nonce at HELLO.
852pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
854pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855/// How long a swap waits for its candidate to register and declare itself
856/// ready when the operator does not say. A module warming as a swap candidate
857/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
858/// daemon allows that plus time to start the process and send HELLO.
859pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861/// Bounded restart policy for crash exits.
862///
863/// `max_restarts` is the number of replacement processes allowed after the
864/// initial spawn WITHIN `window`. After that many crash restarts inside one
865/// window the module enters [`ModuleState::Failed`] and the supervisor stops
866/// the crash loop.
867///
868/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
869/// and that only survived because crashes were rare: a module that crashed
870/// three times across a week was disabled forever by crashes that had nothing
871/// to do with each other. That stopped being survivable once modules began
872/// exiting non-zero whenever the daemon's connection to them drops, because
873/// then every daemon-side connection drop spends a unit of the same budget and
874/// one flappy hour permanently stops a healthy module. Restarts older than
875/// `window` release their slot, so a module that crashed twice yesterday has a
876/// full budget today, while a genuine crash loop -- which is fast by
877/// definition -- still reaches the cap and stops.
878#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880    pub max_restarts: u32,
881    /// Base delay before a crash replacement. The actual delay escalates with
882    /// the number of recent crash replacements and is capped by `max_backoff`.
883    pub backoff: Duration,
884    /// Maximum delay before a crash replacement.
885    pub max_backoff: Duration,
886    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
887    /// budget effectively infinite (nothing is ever in-window), which is why
888    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
889    pub window: Duration,
890}
891
892impl RestartPolicy {
893    /// A policy with the default crash window. Callers that care about the
894    /// window say so with [`Self::with_window`]; the ones that do not are
895    /// asking for the standard rate limit, not for no limit.
896    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897        Self {
898            max_restarts,
899            backoff,
900            max_backoff: DEFAULT_MAX_BACKOFF,
901            window: DEFAULT_RESTART_WINDOW,
902        }
903    }
904
905    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906        self.max_backoff = max_backoff;
907        self
908    }
909
910    pub fn with_window(mut self, window: Duration) -> Self {
911        self.window = window;
912        self
913    }
914
915    /// Calculate the capped exponential delay for the next crash replacement.
916    /// `restart_in_window` is zero for the first replacement after an operator
917    /// action (restart, reload, re-enable) cleared the crash ring, or after all
918    /// older crash replacements have aged out of the window.
919    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920        if self.backoff.is_zero() || self.max_backoff.is_zero() {
921            return Duration::ZERO;
922        }
923
924        let mut delay = self.backoff;
925        for _ in 0..restart_in_window {
926            if delay >= self.max_backoff {
927                return self.max_backoff;
928            }
929            delay = delay
930                .checked_mul(10)
931                .unwrap_or(self.max_backoff)
932                .min(self.max_backoff);
933        }
934        delay.min(self.max_backoff)
935    }
936
937    /// The one sentence that explains a budget-exhausted stop, used for both the
938    /// log line and the terminal record so the two cannot drift. It names the
939    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
940    /// exactly what this budget is not.
941    fn budget_exhausted_detail(&self) -> String {
942        format!(
943            "crash budget exhausted: max_restarts={} within window_secs={}",
944            self.max_restarts,
945            self.window.as_secs()
946        )
947    }
948}
949
950impl Default for RestartPolicy {
951    fn default() -> Self {
952        Self {
953            max_restarts: DEFAULT_MAX_RESTARTS,
954            backoff: DEFAULT_BACKOFF,
955            max_backoff: DEFAULT_MAX_BACKOFF,
956            window: DEFAULT_RESTART_WINDOW,
957        }
958    }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963    restart_in_window: u32,
964    delay: Duration,
965}
966
967/// Whether the daemon itself will bring this module back after the exit being
968/// handled: it is enabled AND its in-window crash restarts are below the cap.
969///
970/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
971/// the window are dropped here rather than by a timer, so the count is right
972/// the moment somebody asks and no bookkeeping runs for idle modules.
973fn daemon_will_restart(
974    state: &mut SupervisorSnapshot,
975    policy: &RestartPolicy,
976    now: Instant,
977) -> bool {
978    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988    Report,
989    Restart,
990    Alert,
991}
992
993impl fmt::Display for HealthAction {
994    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995        f.write_str(match self {
996            Self::Report => "report",
997            Self::Restart => "restart",
998            Self::Alert => "alert",
999        })
1000    }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005    /// Optional loopback HTTP endpoint for a managed non-wire process.
1006    /// Changing it applies live on rescan; the process protocol changes only
1007    /// at its next spawn.
1008    pub http: Option<String>,
1009    pub cadence: Duration,
1010    pub deadline: Duration,
1011    pub failure_threshold: u32,
1012    pub on_degraded: HealthAction,
1013    pub on_failing: HealthAction,
1014    pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018    fn default() -> Self {
1019        Self {
1020            http: None,
1021            cadence: DEFAULT_HEALTH_CADENCE,
1022            deadline: DEFAULT_HEALTH_DEADLINE,
1023            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024            on_degraded: HealthAction::Report,
1025            on_failing: HealthAction::Report,
1026            critical: false,
1027        }
1028    }
1029}
1030
1031/// The supervisor's view of one module's health, relayed to clients over
1032/// channel-0 and rendered by `ck health`.
1033///
1034/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1035/// stated here rather than only at the wire type a consumer reads. A reader can
1036/// look up what `None` means; only a writer can silently change it, and the
1037/// writer has no reason to go looking at a downstream contract before editing.
1038///
1039/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1040/// back to `None` on re-registration precisely so a respawned module does not
1041/// carry its predecessor's timestamp — so an old value and an absent one call for
1042/// opposite readings, and anything that defaulted this to a number would make a
1043/// never-probed module indistinguishable from one probed at the epoch.
1044///
1045/// `detail` and `metrics` are `None` when the module published none on this
1046/// probe, which does not mean it reported nothing wrong — it is also the shape
1047/// when the probe never reached it. `last_probe_ms` is what separates those.
1048#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050    pub status: SupervisorHealthStatus,
1051    pub last_probe_ms: Option<u64>,
1052    pub detail: Option<String>,
1053    pub metrics: Option<Value>,
1054    pub consecutive_failures: u32,
1055    /// Number of replies received after a recurring health probe's deadline.
1056    /// Unlike a timeout, every increment proves the module was alive.
1057    pub late_answer_count: u64,
1058    /// End-to-end latency of the newest late reply, measured from probe start.
1059    pub last_late_answer_latency_ms: Option<u64>,
1060    pub last_action: Option<String>,
1061    /// Set together with `last_action`; the pair moves as one, and both being
1062    /// absent means no escalation has ever been taken rather than that the last
1063    /// one succeeded.
1064    pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068    fn default() -> Self {
1069        Self {
1070            status: SupervisorHealthStatus::Unknown,
1071            last_probe_ms: None,
1072            detail: None,
1073            metrics: None,
1074            consecutive_failures: 0,
1075            late_answer_count: 0,
1076            last_late_answer_latency_ms: None,
1077            last_action: None,
1078            last_action_ms: None,
1079        }
1080    }
1081}
1082
1083/// Typed lifecycle state for a supervised module.
1084#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086    Starting,
1087    Running,
1088    Unresponsive,
1089    Restarting,
1090    Draining,
1091    Stopped,
1092    Failed,
1093    Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098        f.write_str(match self {
1099            Self::Starting => "starting",
1100            Self::Running => "running",
1101            Self::Unresponsive => "unresponsive",
1102            Self::Restarting => "restarting",
1103            Self::Draining => "draining",
1104            Self::Stopped => "stopped",
1105            Self::Failed => "failed",
1106            Self::Disabled => "disabled",
1107        })
1108    }
1109}
1110
1111/// Supervisor classification of a child-process exit.
1112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114    Clean,
1115    Crash,
1116    DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120    fn from(kind: ExitKind) -> Self {
1121        match kind {
1122            ExitKind::Clean => Self::Clean,
1123            ExitKind::Crash => Self::Crash,
1124            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125        }
1126    }
1127}
1128
1129/// Exact process identity retained when a supervised module registers its
1130/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133    pub(crate) pid: u32,
1134    pub(crate) start_time: u64,
1135}
1136
1137/// Last observed child exit, if any.
1138#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140    pub kind: ExitKind,
1141    pub code: Option<i32>,
1142    pub signal: Option<i32>,
1143    pub at_ms: u64,
1144}
1145
1146/// Point-in-time module status answerable by subc without forwarding to the
1147/// module process.
1148#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150    pub module_id: String,
1151    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1152    pub waiting_for_previous_pid: Option<u32>,
1153    /// Why the previous instance still prevents a new start.
1154    pub waiting_for_previous_detail: Option<String>,
1155    /// The daemon's Windows per-spawn record: `pipe` after the process it
1156    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1157    /// List/status reports use this record, not the source the module sends
1158    /// in its registration message's manifest.provenance.launch_nonce_source.
1159    pub launch_nonce_source: Option<String>,
1160    pub state: ModuleState,
1161    pub enabled: bool,
1162    pub process_alive: bool,
1163    pub registration_active: bool,
1164    /// The module's declared wire protocol, carried beside `live` because it is
1165    /// what makes `live` readable: the two fields answer one question together.
1166    /// While a process is alive this is its launch declaration, not a later
1167    /// pending-reload edit. When down it is the configured next launch protocol.
1168    pub protocol: ModuleProtocol,
1169    /// Whether the module is serving, under the strongest definition the daemon
1170    /// can assert for its protocol.
1171    ///
1172    /// A subc module must also be REGISTERED: its process being alive says
1173    /// nothing about whether it can take a request. A `protocol: "none"` module
1174    /// never registers, so that term is dropped and this falls back to "enabled,
1175    /// running, and the process the daemon launched is alive" -- which is all
1176    /// the daemon observes about a process that speaks no subc wire. It stays a
1177    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1178    /// rather than printing it bare.
1179    pub live: bool,
1180    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1181    /// restarts have already released their slot, so this count can go down
1182    /// without anybody touching the module.
1183    pub restart_count: u32,
1184    /// Replacement processes spawned over this module's entire supervisor lifetime;
1185    /// unlike `restart_count`, this value is never reset by an operator action
1186    /// and never falls out of a window.
1187    pub lifetime_restarts: u32,
1188    pub spawn_generation: u64,
1189    /// The budget `restart_count` is spent against. Carried alongside the count
1190    /// because the count alone does not say how close the module is to being
1191    /// disabled, and reporting one without the other is what makes an
1192    /// about-to-be-retired module look ordinary.
1193    pub max_restarts: u32,
1194    /// The span `restart_count` is counted over. Carried with the pair above for
1195    /// the same reason they are carried together: "2 of 3" means one thing for a
1196    /// ten-minute window and something else entirely for a lifetime.
1197    pub restart_window: Duration,
1198    /// Effective drain and restart timing policy used by this running module.
1199    /// These values are carried together with the restart budget so status
1200    /// readers can compare configured intent with what the supervisor applied.
1201    pub drain_timeout: Duration,
1202    pub restart_backoff: Duration,
1203    pub restart_max_backoff: Duration,
1204    /// The module's process. On macOS this stays absent while the `ck-subc`
1205    /// launch trampoline is still running in that pid, and appears once the
1206    /// exec acknowledgement confirms the module image has replaced it. Launch
1207    /// time and the supervisor's own hold on the process are unaffected.
1208    pub pid: Option<u32>,
1209    pub spawned_at_ms: Option<u64>,
1210    pub spawned_from: Option<PathBuf>,
1211    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212    pub process_start_time: Option<u64>,
1213    pub last_exit: Option<ExitReport>,
1214    pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220    generation: u64,
1221    turn: u64,
1222    registered_connection: Option<ConnectionId>,
1223    next_probe_at: Option<Instant>,
1224    wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230    #[cfg(windows)]
1231    pipe_consumed: Option<Arc<OnceLock<()>>>,
1232    /// Counts running-child loop turns, not executor polls of a parked wait.
1233    #[cfg(test)]
1234    actor_turns: u64,
1235    /// Running select after exec confirmation and with no unconsumed registry
1236    /// event. Recording a spawn or starting a loop turn is not this barrier.
1237    #[cfg(test)]
1238    actor_select: Option<ActorSelectCheckpoint>,
1239    state: ModuleState,
1240    enabled: bool,
1241    process_alive: bool,
1242    spawned_protocol: Option<ModuleProtocol>,
1243    spawn_failure: Option<String>,
1244    waiting_for_previous_pid: Option<u32>,
1245    waiting_for_previous_detail: Option<String>,
1246    /// When each crash restart was spent, oldest first. This IS the crash
1247    /// budget: its in-window length is the count an operator sees and the count
1248    /// the restart decision is made against, so there is no second counter that
1249    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1250    /// operator actions that used to zero the old lifetime counter.
1251    crash_restarts: VecDeque<Instant>,
1252    lifetime_restarts: u32,
1253    /// Successful child spawns in this daemon incarnation.
1254    ///
1255    /// `lifetime_restarts` was considered and rejected: it starts at zero
1256    /// (line 640), successful initial/operator spawns in `set_running` do not
1257    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1258    /// increments before a successful replacement exists (lines 604, 3846,
1259    /// and 3921), so a failed spawn can consume it. This counter moves only
1260    /// when a live PID is accepted below.
1261    spawn_generation: u64,
1262    pid: Option<u32>,
1263    #[cfg(target_os = "macos")]
1264    report_ready: Option<Arc<OnceLock<()>>>,
1265    /// Last reaped child, retained after current process facts are cleared.
1266    reaped_pid: Option<u32>,
1267    /// Whether a replacement has already been scheduled for the current process.
1268    respawn_pending: bool,
1269    /// A second restart is waiting for the replacement already scheduled.
1270    coalesced_restart_pending: bool,
1271    spawned_at_ms: Option<u64>,
1272    spawned_from: Option<PathBuf>,
1273    launch_spec_digest: Option<LaunchSpecDigest>,
1274    spawned_file_identity: Option<SpawnedFileIdentity>,
1275    process_start_time: Option<u64>,
1276    deliberate_severance: Option<ProcessIdentity>,
1277    last_exit: Option<ExitReport>,
1278    /// Diagnostic attached to the next drain's terminal record, if any.
1279    drain_disposition_detail: Option<String>,
1280    health: ModuleHealthStatus,
1281    /// Whether the current process was started as a swap candidate and so
1282    /// lives in the module's alternate cgroup. The next swap's candidate takes
1283    /// the other one, so the two processes of a swap never share a cgroup. A
1284    /// plain spawn always uses the primary cgroup.
1285    in_alternate_slot: bool,
1286    /// Whether the current `Draining` state ends in a replacement process
1287    /// (restart, reload, health restart) rather than a stop. Only meaningful
1288    /// while `state` is `Draining`; every entry into that state rewrites it.
1289    /// It is what lets route.open answer the retryable `module_reloading` to a
1290    /// consumer that reaches a still-registered process mid-restart, instead of
1291    /// the `supervisor_not_live` a stop or disable deserves.
1292    draining_to_replace: bool,
1293    /// Whether a configuration update has been applied since the current
1294    /// process was spawned, so that process runs an older spec than the one
1295    /// the supervisor now holds. A queued restart is only coalesced into a
1296    /// fresher process when this is false: a restart requested to pick up a
1297    /// new configuration must not be satisfied by a process that predates it.
1298    configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302    /// The pid that status, provenance and resource readings may report. While
1303    /// the launch trampoline still runs in the pid, reading its executable or
1304    /// resource use would describe `ck-subc`, not the module, so none is
1305    /// reported until the exec acknowledgement confirms the module image.
1306    fn reported_pid(&self) -> Option<u32> {
1307        #[cfg(target_os = "macos")]
1308        if self
1309            .report_ready
1310            .as_ref()
1311            .is_some_and(|ready| ready.get().is_none())
1312        {
1313            return None;
1314        }
1315        self.pid
1316    }
1317
1318    fn starting() -> Self {
1319        Self::new(ModuleState::Starting, true)
1320    }
1321
1322    fn disabled() -> Self {
1323        Self::new(ModuleState::Disabled, false)
1324    }
1325
1326    /// Crash restarts still inside `window`, having dropped the ones that are
1327    /// not. Pruning on read is what makes the budget a rate: an instant older
1328    /// than the window stops holding a slot the moment anybody counts.
1329    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330        while let Some(oldest) = self.crash_restarts.front() {
1331            if now.duration_since(*oldest) > window {
1332                self.crash_restarts.pop_front();
1333            } else {
1334                break;
1335            }
1336        }
1337        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338    }
1339
1340    /// Spend one unit of the crash budget and record the restart in the ledger.
1341    ///
1342    /// The ring is bounded by the cap because more than `max_restarts` in-window
1343    /// instants can never be reached (the caller refuses the restart first), so
1344    /// anything beyond that is an unbounded queue waiting to happen.
1345    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346        self.crash_restarts.push_back(now);
1347        while self.crash_restarts.len() > policy.max_restarts as usize {
1348            self.crash_restarts.pop_front();
1349        }
1350        self.lifetime_restarts += 1;
1351    }
1352
1353    /// Reserve one crash-restart slot and calculate the delay before respawning.
1354    /// The count is captured before recording this restart, so the first retry
1355    /// uses the base delay and each later in-window retry escalates once.
1356    fn next_crash_restart(
1357        &mut self,
1358        policy: &RestartPolicy,
1359        now: Instant,
1360    ) -> Option<CrashRestartSchedule> {
1361        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362        if restart_in_window >= policy.max_restarts {
1363            return None;
1364        }
1365        self.record_crash_restart(policy, now);
1366        Some(CrashRestartSchedule {
1367            restart_in_window,
1368            delay: policy.delay_for_restart(restart_in_window),
1369        })
1370    }
1371
1372    /// Give the module its full budget back, as an operator restart, reload, or
1373    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1374    /// ledger of what actually happened, and an operator action does not unmake
1375    /// the crashes.
1376    fn clear_crash_restarts(&mut self) {
1377        self.crash_restarts.clear();
1378    }
1379
1380    fn new(state: ModuleState, enabled: bool) -> Self {
1381        Self {
1382            exec_outcome: None,
1383            #[cfg(test)]
1384            actor_turns: 0,
1385            #[cfg(test)]
1386            actor_select: None,
1387            state,
1388            enabled,
1389            process_alive: false,
1390            #[cfg(windows)]
1391            pipe_consumed: None,
1392            spawned_protocol: None,
1393            spawn_failure: None,
1394            waiting_for_previous_pid: None,
1395            waiting_for_previous_detail: None,
1396            crash_restarts: VecDeque::new(),
1397            lifetime_restarts: 0,
1398            spawn_generation: 0,
1399            pid: None,
1400            #[cfg(target_os = "macos")]
1401            report_ready: None,
1402            reaped_pid: None,
1403            respawn_pending: false,
1404            coalesced_restart_pending: false,
1405            spawned_at_ms: None,
1406            spawned_from: None,
1407            launch_spec_digest: None,
1408            spawned_file_identity: None,
1409            process_start_time: None,
1410            deliberate_severance: None,
1411            last_exit: None,
1412            drain_disposition_detail: None,
1413            health: ModuleHealthStatus::default(),
1414            in_alternate_slot: false,
1415            draining_to_replace: false,
1416            configuration_updated_since_spawn: false,
1417        }
1418    }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427    version: u8,
1428    frames: mpsc::Sender<Frame>,
1429    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1430    /// from which event. The full frame channel cannot carry that news, so it
1431    /// travels beside it; see `SpawnEventFeed::subscribe`.
1432    lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437    daemon_incarnation: String,
1438    seq: u64,
1439    capacity: usize,
1440    live: HashMap<String, LiveSpawn>,
1441    generations: HashMap<String, u64>,
1442    events: VecDeque<SpawnEvent>,
1443    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447    fn default() -> Self {
1448        Self {
1449            daemon_incarnation: "unconfigured".to_string(),
1450            seq: 0,
1451            capacity: SPAWN_EVENT_RING_CAPACITY,
1452            live: HashMap::new(),
1453            generations: HashMap::new(),
1454            events: VecDeque::new(),
1455            subscribers: HashMap::new(),
1456        }
1457    }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462    Arc<Mutex<SpawnEventState>>,
1463    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468    ForeignIncarnation { current: String },
1469    TooOld { oldest: SpawnCursor },
1470    Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474    fn configure_incarnation(&self, daemon_incarnation: String) {
1475        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476        state.daemon_incarnation = daemon_incarnation;
1477        state.seq = 0;
1478        state.live.clear();
1479        state.generations.clear();
1480        state.events.clear();
1481        state.subscribers.clear();
1482    }
1483
1484    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485        SpawnCursor {
1486            daemon_incarnation: state.daemon_incarnation.clone(),
1487            seq: state.seq,
1488        }
1489    }
1490
1491    fn snapshot(&self) -> SpawnSnapshot {
1492        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495        SpawnSnapshot {
1496            cursor: Self::cursor(&state),
1497            ring_bound: state.capacity as u64,
1498            live,
1499        }
1500    }
1501
1502    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504        let generation = state
1505            .generations
1506            .get(module_id)
1507            .copied()
1508            .unwrap_or(0)
1509            .checked_add(1)
1510            .expect("spawn generation exhausted");
1511        state.generations.insert(module_id.to_string(), generation);
1512        let live = LiveSpawn {
1513            module_id: module_id.to_string(),
1514            spawn_generation: generation,
1515            pid,
1516            spawned_at_ms,
1517        };
1518        state.live.insert(module_id.to_string(), live);
1519        Self::emit_locked(
1520            &mut state,
1521            SpawnEventKind::Spawned,
1522            module_id.to_string(),
1523            generation,
1524            pid,
1525            None,
1526            None,
1527        );
1528        generation
1529    }
1530
1531    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533        let Some(live) = state.live.remove(module_id) else {
1534            warn!(
1535                module_id,
1536                "terminal record had no live spawn event identity"
1537            );
1538            return;
1539        };
1540        Self::emit_locked(
1541            &mut state,
1542            SpawnEventKind::Exited,
1543            module_id.to_string(),
1544            live.spawn_generation,
1545            live.pid,
1546            exit_code,
1547            exit_signal,
1548        );
1549    }
1550
1551    /// Report the exit of a process that a swap has already replaced.
1552    ///
1553    /// `emit_exited` removes the module's live entry, which after a swap's
1554    /// cutover describes the promoted candidate, not the old process now
1555    /// exiting. This emits the old generation's exit and leaves the live entry
1556    /// alone unless it still names that generation.
1557    fn emit_superseded_exited(
1558        &self,
1559        module_id: &str,
1560        spawn_generation: u64,
1561        pid: u32,
1562        exit_code: Option<i32>,
1563        exit_signal: Option<i32>,
1564    ) {
1565        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566        if state
1567            .live
1568            .get(module_id)
1569            .is_some_and(|live| live.spawn_generation == spawn_generation)
1570        {
1571            state.live.remove(module_id);
1572        }
1573        Self::emit_locked(
1574            &mut state,
1575            SpawnEventKind::Exited,
1576            module_id.to_string(),
1577            spawn_generation,
1578            pid,
1579            exit_code,
1580            exit_signal,
1581        );
1582    }
1583
1584    #[allow(clippy::too_many_arguments)]
1585    fn emit_locked(
1586        state: &mut SpawnEventState,
1587        kind: SpawnEventKind,
1588        module_id: String,
1589        spawn_generation: u64,
1590        pid: u32,
1591        exit_code: Option<i32>,
1592        exit_signal: Option<i32>,
1593    ) {
1594        state.seq = state
1595            .seq
1596            .checked_add(1)
1597            .expect("spawn event sequence exhausted");
1598        let event = SpawnEvent {
1599            cursor: Self::cursor(state),
1600            kind,
1601            module_id,
1602            spawn_generation,
1603            pid,
1604            exit_code,
1605            exit_signal,
1606        };
1607        state.events.push_back(event.clone());
1608        while state.events.len() > state.capacity {
1609            state.events.pop_front();
1610        }
1611        let body = match serde_json::to_vec(&event) {
1612            Ok(body) => body,
1613            Err(error) => {
1614                error!(%error, "failed to serialize supervisor spawn event");
1615                return;
1616            }
1617        };
1618        state.subscribers.retain(|(connection_id, corr), subscriber| {
1619            let frame = Frame::build_with_version(
1620                subscriber.version,
1621                FrameType::StreamData,
1622                control_flags(),
1623                0,
1624                0,
1625                *corr,
1626                body.clone(),
1627            );
1628            match frame {
1629                Ok(frame) => {
1630                    if subscriber.frames.try_send(frame).is_ok() {
1631                        true
1632                    } else {
1633                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634                        if let Some(lagged) = subscriber.lagged.take() {
1635                            let _ = lagged.send(event.cursor.clone());
1636                        }
1637                        false
1638                    }
1639                }
1640                Err(error) => {
1641                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642                    false
1643                }
1644            }
1645        });
1646    }
1647
1648    fn subscribe(
1649        &self,
1650        connection_id: ConnectionId,
1651        corr: u64,
1652        version: u8,
1653        since: Option<SpawnCursor>,
1654        sink: FrameSink,
1655    ) -> Result<(), SpawnSubscribeRefusal> {
1656        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658        {
1659            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660            let replay = if let Some(since) = since {
1661                if since.daemon_incarnation != state.daemon_incarnation {
1662                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663                        current: state.daemon_incarnation.clone(),
1664                    });
1665                }
1666                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667                    if since.seq < oldest.seq.saturating_sub(1) {
1668                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669                    }
1670                }
1671                state
1672                    .events
1673                    .iter()
1674                    .filter(|event| event.cursor.seq > since.seq)
1675                    .cloned()
1676                    .collect::<Vec<_>>()
1677            } else {
1678                Vec::new()
1679            };
1680            for event in replay {
1681                let body = serde_json::to_vec(&event)
1682                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683                let frame = Frame::build_with_version(
1684                    version,
1685                    FrameType::StreamData,
1686                    control_flags(),
1687                    0,
1688                    0,
1689                    corr,
1690                    body,
1691                )
1692                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693                frames
1694                    .try_send(frame)
1695                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696            }
1697            state.subscribers.insert(
1698                (connection_id, corr),
1699                SpawnSubscriber {
1700                    version,
1701                    frames,
1702                    lagged: Some(lagged),
1703                },
1704            );
1705        }
1706        // The lagged terminal is sent here, by the forwarder, rather than by
1707        // the emitter: at the moment of the drop the subscriber's own channel
1708        // is full, and writing to the connection sink directly from the emitter
1709        // would put the Error AHEAD of the events still queued in that channel
1710        // (and the emitter holds the feed lock, so it cannot await the sink).
1711        // Dropping the subscriber drops the only sender, so `recv` drains every
1712        // queued event and then returns `None`; only then is the Error sent, so
1713        // the client sees each event it can keep, then the reason it was cut.
1714        // Cancel and connection removal drop the oneshot unsent, so they end
1715        // the stream with no Error.
1716        tokio::spawn(async move {
1717            while let Some(frame) = receiver.recv().await {
1718                if sink.send(frame).await.is_err() {
1719                    return;
1720                }
1721            }
1722            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723                return;
1724            };
1725            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726                Ok(frame) => {
1727                    let _ = sink.send(frame).await;
1728                }
1729                Err(error) => {
1730                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731                }
1732            }
1733        });
1734        Ok(())
1735    }
1736
1737    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738        let Some(subscriber) = self
1739            .0
1740            .lock()
1741            .unwrap_or_else(|p| p.into_inner())
1742            .subscribers
1743            .remove(&(connection_id, corr))
1744        else {
1745            return false;
1746        };
1747        if let Ok(frame) = Frame::build_with_version(
1748            subscriber.version,
1749            FrameType::StreamEnd,
1750            control_flags(),
1751            0,
1752            0,
1753            corr,
1754            Vec::new(),
1755        ) {
1756            tokio::spawn(async move {
1757                let _ = subscriber.frames.send(frame).await;
1758            });
1759        }
1760        true
1761    }
1762
1763    fn remove_connection(&self, connection_id: ConnectionId) {
1764        self.0
1765            .lock()
1766            .unwrap_or_else(|p| p.into_inner())
1767            .subscribers
1768            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769    }
1770
1771    #[cfg(any(test, feature = "test-support"))]
1772    fn set_capacity(&self, capacity: usize) {
1773        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774    }
1775
1776    #[cfg(any(test, feature = "test-support"))]
1777    fn subscriber_count(&self) -> usize {
1778        self.0
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .subscribers
1782            .len()
1783    }
1784}
1785
1786/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1787/// The terminal Error a lagged spawn subscriber receives after its queued events.
1788fn spawn_subscriber_lagged_frame(
1789    version: u8,
1790    corr: u64,
1791    first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796            .to_string(),
1797        detail: Some(serde_json::json!({
1798            "first_undelivered_cursor": first_undelivered
1799        })),
1800    })
1801    .map_err(|error| error.to_string())?;
1802    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803        .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807    fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809    /// Whether the supervisor is replacing this module's process right now: an
1810    /// operator restart or reload, a health restart, or a crash respawn whose
1811    /// backoff is running. A module in that state is not live, but a consumer
1812    /// refused now should retry shortly rather than treat the target as gone.
1813    /// Stopped, failed, and disabled modules are not replacing.
1814    fn process_replacing(&self, _module_id: &str) -> bool {
1815        false
1816    }
1817}
1818
1819/// Shared process-liveness registry keyed by supervised `module_id`.
1820#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826    pub fn new() -> Self {
1827        Self::default()
1828    }
1829
1830    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831        let mut snapshots = self
1832            .snapshots
1833            .lock()
1834            .unwrap_or_else(|poisoned| poisoned.into_inner());
1835        snapshots.insert(module_id, snapshot);
1836    }
1837
1838    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        let is_current = snapshots
1844            .get(module_id)
1845            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846            .unwrap_or(false);
1847        if is_current {
1848            snapshots.remove(module_id);
1849        }
1850    }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854    fn process_live(&self, module_id: &str) -> Option<bool> {
1855        let snapshot = {
1856            let snapshots = self
1857                .snapshots
1858                .lock()
1859                .unwrap_or_else(|poisoned| poisoned.into_inner());
1860            snapshots.get(module_id).cloned()
1861        }?;
1862        let snapshot = snapshot
1863            .lock()
1864            .unwrap_or_else(|poisoned| poisoned.into_inner());
1865        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866    }
1867
1868    fn process_replacing(&self, module_id: &str) -> bool {
1869        let Some(snapshot) = self
1870            .snapshots
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner())
1873            .get(module_id)
1874            .cloned()
1875        else {
1876            return false;
1877        };
1878        let snapshot = snapshot
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner());
1881        snapshot.enabled
1882            && match snapshot.state {
1883                ModuleState::Restarting => true,
1884                ModuleState::Draining => snapshot.draining_to_replace,
1885                ModuleState::Starting
1886                | ModuleState::Running
1887                | ModuleState::Unresponsive
1888                | ModuleState::Stopped
1889                | ModuleState::Failed
1890                | ModuleState::Disabled => false,
1891            }
1892    }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898    reached: tokio::sync::Notify,
1899    resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904    Spawn,
1905    Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910    deadline: Instant,
1911    kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918    start_requester: Option<Principal>,
1919    pending_activity_restart: bool,
1920    provenance_probe: ExecutableIdentityProbe,
1921    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1922    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923    /// A reload acknowledges completion only after its replacement registers.
1924    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925    restart_policy: RestartPolicy,
1926    /// This module's RESOLVED drain budget: per-module config when present,
1927    /// else `default_drain_timeout`.
1928    drain_timeout: Duration,
1929    /// Shared with the status handle so the attested value changes atomically
1930    /// when a rescan updates the running drain policy.
1931    effective_drain_timeout: Arc<Mutex<Duration>>,
1932    /// The supervisor-wide fallback, kept so a configuration update that
1933    /// REMOVES the per-module override can re-resolve to it.
1934    default_drain_timeout: Duration,
1935    health: HealthConfig,
1936    connection_file_path: Option<PathBuf>,
1937    capture_logs_dir: Option<PathBuf>,
1938    forwarding: Option<Arc<ForwardingTable>>,
1939    /// The shared handle, so every spawn path (initial, restart, reload) records the
1940    /// reserved-module launch nonce the HELLO verifier checks against.
1941    supervisor_handle: Option<SupervisorHandle>,
1942    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1943    /// status queries.
1944    ///
1945    /// One ring per module, held across every respawn. The lines explaining an exit
1946    /// are written BEFORE that exit, so a ring recreated per process would be empty
1947    /// exactly when it is asked for.
1948    stderr_ring: Arc<Mutex<StderrRing>>,
1949    terminal_ring: Arc<Mutex<TerminalRing>>,
1950    spawn_events: SpawnEventFeed,
1951    child_roster: ChildRoster,
1952    #[cfg(target_os = "linux")]
1953    cgroup_placement: Option<subc_cgroup::Placement>,
1954    #[cfg(test)]
1955    test_seed_stale_facts_before_enable_spawn: bool,
1956    #[cfg(test)]
1957    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962    spec: ModuleSpec,
1963    health: HealthConfig,
1964}
1965
1966/// Shared daemon lookup table for supervised module handles.
1967///
1968/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1969/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1970/// launch nonces recorded at spawn are checked by the same daemon instance.
1971#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976    /// Module ids the supervisor has taken on. An id is added BEFORE the
1977    /// module's first process is spawned and removed only when the module
1978    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1979    /// the keys of `modules`.
1980    ///
1981    /// `modules` cannot answer "is this module configured?" on its own: a
1982    /// [`SupervisedModule`] only exists once its process has been spawned, and
1983    /// a fast child can connect, register, sync its scopes and ask about them
1984    /// before the supervisor has inserted it. Answering "not configured" in that
1985    /// gap makes scope admission refuse with the terminal "will never sync"
1986    /// instead of the retryable "has not synced yet".
1987    configured_ids: Arc<Mutex<HashSet<String>>>,
1988    spawn_events: SpawnEventFeed,
1989    /// The current expected launch nonce for each reserved module_id. Set when the
1990    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1991    /// non-reserved module never has an entry here and is never nonce-checked.
1992    /// Reserved module ids and the nonce that authorizes their next HELLO.
1993    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1994    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1995    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1996    /// had NO entry and admitted anyone: the reservation protected the nonce
1997    /// holder, not the NAME (found live by CKCRED's canary probe registering
1998    /// against a reserved scratch id).
1999    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2001    ///
2002    /// This is deliberately in-memory only: subc is state-free across daemon
2003    /// restarts, and the tombstone only explains the hours-after-removal window
2004    /// while this executing daemon is still alive. Do not persist it in a store.
2005    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006    /// The current launch nonce for every supervised spawn. This is separate from
2007    /// reserved_nonces because consumer route.open attestation applies to all spawned
2008    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2009    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010    /// Reserved namespace prefixes mapped to the supervised owner module whose
2011    /// current spawn nonce authorizes HELLO claims below the prefix.
2012    ///
2013    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2014    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2015    /// accidental collisions and lower-trust processes from squatting protected
2016    /// namespaces.
2017    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018    /// Blue/green swaps in progress, by module id. An entry exists from just
2019    /// before the candidate process is spawned until the swap has failed, or
2020    /// has cut over and the old process is gone. While it exists, HELLO for the
2021    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2022    /// consumer attestation accepts both processes' nonces.
2023    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2025    promotion_observer: PromotionObserverSlot,
2026    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2027    /// this daemon-wide ordering, a rescan could retire or update a module while a
2028    /// concurrent reload still held its old handle and launch specification.
2029    operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032/// Told when a swap has promoted its candidate to be the module's active
2033/// registration.
2034///
2035/// An ordinary HELLO runs the control plane's registration side effects (the
2036/// capability cache, the deny census, the requirement recompute) as it
2037/// registers. A swap candidate's HELLO does not, because it is not routable;
2038/// promotion is when those must run instead, and promotion happens in the
2039/// supervisor, which has no other way into the control handler.
2040pub(crate) trait SwapPromotionObserver: Send + Sync {
2041    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2045/// control handler) owns this handle, so a strong reference back would be a
2046/// cycle that keeps both alive.
2047#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052        f.write_str("PromotionObserverSlot")
2053    }
2054}
2055
2056/// The nonces of one open swap.
2057#[derive(Debug, Clone)]
2058struct OpenSwap {
2059    /// The launch nonce minted for the candidate process. It is the swap
2060    /// token: the only thing that admits a HELLO into the candidate slot.
2061    candidate_nonce: String,
2062    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2063    /// here because cutover moves the module's recorded spawn nonce to the
2064    /// candidate while the incumbent is still draining and its consumers are
2065    /// still attesting with this one.
2066    incumbent_nonce: Option<String>,
2067    /// Set once a HELLO has been admitted with the swap token, so the token
2068    /// admits one registration and cannot be replayed after cutover empties
2069    /// the candidate slot.
2070    candidate_admitted: bool,
2071}
2072
2073/// What the swap gate says about a HELLO. See
2074/// [`SupervisorHandle::swap_hello_admission`].
2075#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077    /// No swap is open for the id (or the HELLO carries the incumbent's own
2078    /// nonce); the ordinary gates decide.
2079    NotSwapping,
2080    /// The HELLO carries the swap token: register it into the candidate slot.
2081    Candidate,
2082    /// A swap is open and the HELLO carries a nonce the supervisor did not
2083    /// mint for this id, no nonce, or a token already used.
2084    Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089    Exact {
2090        module_id: String,
2091    },
2092    Prefix {
2093        prefix: String,
2094        owner_module_id: String,
2095    },
2096}
2097
2098impl SupervisorHandle {
2099    pub fn new() -> Self {
2100        Self::default()
2101    }
2102
2103    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104        self.spawn_events.snapshot()
2105    }
2106
2107    pub(crate) fn subscribe_spawns(
2108        &self,
2109        connection_id: ConnectionId,
2110        corr: u64,
2111        version: u8,
2112        since: Option<SpawnCursor>,
2113        sink: FrameSink,
2114    ) -> Result<(), SpawnSubscribeRefusal> {
2115        self.spawn_events
2116            .subscribe(connection_id, corr, version, since, sink)
2117    }
2118
2119    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120        self.spawn_events.cancel(connection_id, corr)
2121    }
2122
2123    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124        self.spawn_events.remove_connection(connection_id);
2125    }
2126
2127    #[cfg(any(test, feature = "test-support"))]
2128    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129        assert!(capacity > 0, "spawn event capacity must be non-zero");
2130        self.spawn_events.set_capacity(capacity);
2131    }
2132
2133    #[cfg(any(test, feature = "test-support"))]
2134    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135        self.spawn_events.subscriber_count()
2136    }
2137
2138    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2139    /// a respawn invalidates stale consumer identities.
2140    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141        self.spawn_nonces
2142            .lock()
2143            .unwrap_or_else(|poisoned| poisoned.into_inner())
2144            .insert(module_id.to_string(), nonce);
2145    }
2146
2147    /// Record the launch nonce expected from the next HELLO for a reserved module,
2148    /// replacing any prior nonce (a respawn invalidates the previous one).
2149    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150        self.reserved_nonces
2151            .lock()
2152            .unwrap_or_else(|poisoned| poisoned.into_inner())
2153            .insert(module_id.to_string(), Some(nonce));
2154    }
2155
2156    /// Record namespace prefixes owned by a supervised module.
2157    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158        let mut owners = self
2159            .reserved_prefix_owners
2160            .lock()
2161            .unwrap_or_else(|poisoned| poisoned.into_inner());
2162        owners.retain(|_, owner| owner != owner_module_id);
2163        for prefix in prefixes {
2164            owners.insert(prefix.clone(), owner_module_id.to_string());
2165        }
2166    }
2167
2168    /// The launch nonce most recently minted for a module's spawn, if any.
2169    #[cfg(test)]
2170    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171        self.spawn_nonces
2172            .lock()
2173            .unwrap_or_else(|poisoned| poisoned.into_inner())
2174            .get(module_id)
2175            .cloned()
2176    }
2177
2178    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180        let spawn_nonce = self
2181            .spawn_nonces
2182            .lock()
2183            .unwrap_or_else(|poisoned| poisoned.into_inner())
2184            .get(&spec.module_id)
2185            .cloned();
2186        let mut reserved_nonces = self
2187            .reserved_nonces
2188            .lock()
2189            .unwrap_or_else(|poisoned| poisoned.into_inner());
2190        if spec.reserved {
2191            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2192            // reserved name whose module has never spawned has no legitimate
2193            // holder, and the entry's absence is what used to leave the name
2194            // open to the first claimant.
2195            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196        }
2197        drop(reserved_nonces);
2198        // A later unreserved declaration must not silently unreserve an id that
2199        // was retained after its reserved configuration was removed. The explicit
2200        // release ceremony is the only operation that retires that gate.
2201        self.removal_tombstones
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .remove(&spec.module_id);
2205    }
2206
2207    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2208    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2209    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2210    /// with no matching prefix are always authorized.
2211    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212        self.reserved_hello_rejection(module_id, presented)
2213            .is_none()
2214    }
2215
2216    pub(crate) fn reserved_hello_rejection(
2217        &self,
2218        module_id: &str,
2219        presented: Option<&str>,
2220    ) -> Option<ReservedHelloRejection> {
2221        let nonces = self
2222            .reserved_nonces
2223            .lock()
2224            .unwrap_or_else(|poisoned| poisoned.into_inner());
2225        if let Some(expected) = nonces.get(module_id) {
2226            // `None` = reserved with no legitimate holder: refuse every
2227            // presentation, because no process can hold a nonce that was never
2228            // minted. Only a real minted nonce admits, in constant time.
2229            let authorized = match expected {
2230                Some(expected) => {
2231                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232                }
2233                None => false,
2234            };
2235            if authorized {
2236                return None;
2237            }
2238            return Some(ReservedHelloRejection::Exact {
2239                module_id: module_id.to_string(),
2240            });
2241        }
2242        drop(nonces);
2243
2244        let matched_prefix = self
2245            .reserved_prefix_owners
2246            .lock()
2247            .unwrap_or_else(|poisoned| poisoned.into_inner())
2248            .iter()
2249            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250            .max_by_key(|(prefix, _)| prefix.len())
2251            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252        let (prefix, owner_module_id) = matched_prefix?;
2253
2254        let authorized = presented.is_some_and(|presented| {
2255            self.spawn_nonces
2256                .lock()
2257                .unwrap_or_else(|poisoned| poisoned.into_inner())
2258                .get(&owner_module_id)
2259                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260                // While the owner is being swapped, children started by
2261                // either of its two processes hold that process's nonce.
2262                || self.swap_nonce_matches(&owner_module_id, presented)
2263        });
2264        if authorized {
2265            None
2266        } else {
2267            Some(ReservedHelloRejection::Prefix {
2268                prefix,
2269                owner_module_id,
2270            })
2271        }
2272    }
2273
2274    /// Whether a consumer connection proved it came from a daemon-spawned module.
2275    ///
2276    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2277    /// accepted only for module ids the supervisor has spawned.
2278    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279        if presented.is_empty() {
2280            return false;
2281        }
2282        let nonces = self
2283            .spawn_nonces
2284            .lock()
2285            .unwrap_or_else(|poisoned| poisoned.into_inner());
2286        let current = nonces
2287            .get(module_id)
2288            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289        drop(nonces);
2290        // During a swap two processes of the module are alive, and a consumer
2291        // started by either one presents that process's nonce. Accepting only
2292        // the recorded one would fail the incumbent's consumers for the whole
2293        // overlap once cutover moves the record to the candidate.
2294        current || self.swap_nonce_matches(module_id, presented)
2295    }
2296
2297    /// Whether `presented` is either nonce of an open swap for `module_id`.
2298    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299        let swaps = self
2300            .swaps
2301            .lock()
2302            .unwrap_or_else(|poisoned| poisoned.into_inner());
2303        swaps.get(module_id).is_some_and(|swap| {
2304            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307                })
2308        })
2309    }
2310
2311    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2312    /// Called before the candidate process exists.
2313    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314        let incumbent_nonce = self
2315            .spawn_nonces
2316            .lock()
2317            .unwrap_or_else(|poisoned| poisoned.into_inner())
2318            .get(module_id)
2319            .cloned();
2320        self.swaps
2321            .lock()
2322            .unwrap_or_else(|poisoned| poisoned.into_inner())
2323            .insert(
2324                module_id.to_string(),
2325                OpenSwap {
2326                    candidate_nonce,
2327                    incumbent_nonce,
2328                    candidate_admitted: false,
2329                },
2330            );
2331    }
2332
2333    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2334    /// the module's recorded one.
2335    pub(crate) fn close_swap(&self, module_id: &str) {
2336        self.swaps
2337            .lock()
2338            .unwrap_or_else(|poisoned| poisoned.into_inner())
2339            .remove(module_id);
2340    }
2341
2342    /// Install the observer told about swap promotions, replacing any earlier
2343    /// one.
2344    pub(crate) fn set_swap_promotion_observer(
2345        &self,
2346        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347    ) {
2348        *self
2349            .promotion_observer
2350            .0
2351            .lock()
2352            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353    }
2354
2355    /// Tell the installed observer, if it is still alive, that a swap promoted
2356    /// `registration`.
2357    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358        let observer = self
2359            .promotion_observer
2360            .0
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner())
2363            .as_ref()
2364            .and_then(std::sync::Weak::upgrade);
2365        if let Some(observer) = observer {
2366            observer.swap_promoted(registration);
2367        }
2368    }
2369
2370    /// Whether a swap is open for `module_id`.
2371    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372        self.swaps
2373            .lock()
2374            .unwrap_or_else(|poisoned| poisoned.into_inner())
2375            .contains_key(module_id)
2376    }
2377
2378    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2379    /// respawn would, once cutover has made the candidate the module's process.
2380    /// The swap stays open so the incumbent's nonce keeps attesting until the
2381    /// incumbent has drained and exited.
2382    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383        let candidate_nonce = self
2384            .swaps
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .get(module_id)
2388            .map(|swap| swap.candidate_nonce.clone());
2389        let Some(nonce) = candidate_nonce else {
2390            return;
2391        };
2392        self.set_spawn_nonce(module_id, nonce.clone());
2393        if reserved {
2394            self.set_reserved_nonce(module_id, nonce);
2395        }
2396    }
2397
2398    /// The swap gate for a HELLO claiming `module_id`.
2399    ///
2400    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2401    /// presents the candidate nonce, which the reserved gate (holding the
2402    /// incumbent's nonce) would refuse as `reserved_module` before swap
2403    /// admission was ever reached. And it applies to unreserved ids too: for an
2404    /// unreserved id the only thing that ever stopped a second process claiming
2405    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2406    /// refusal a swap lifts for its candidate.
2407    ///
2408    /// The incumbent's own nonce falls through to the ordinary gates, which
2409    /// treat it as they always have (a live incumbent is refused as a
2410    /// duplicate). Anything else while a swap is open is refused, including an
2411    /// absent nonce.
2412    pub(crate) fn swap_hello_admission(
2413        &self,
2414        module_id: &str,
2415        presented: Option<&str>,
2416    ) -> SwapHelloAdmission {
2417        let swaps = self
2418            .swaps
2419            .lock()
2420            .unwrap_or_else(|poisoned| poisoned.into_inner());
2421        let Some(swap) = swaps.get(module_id) else {
2422            return SwapHelloAdmission::NotSwapping;
2423        };
2424        let Some(presented) = presented else {
2425            return SwapHelloAdmission::Refused;
2426        };
2427        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428            return if swap.candidate_admitted {
2429                SwapHelloAdmission::Refused
2430            } else {
2431                SwapHelloAdmission::Candidate
2432            };
2433        }
2434        if swap
2435            .incumbent_nonce
2436            .as_deref()
2437            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438        {
2439            return SwapHelloAdmission::NotSwapping;
2440        }
2441        SwapHelloAdmission::Refused
2442    }
2443
2444    /// Record that the swap token has registered a candidate, so it admits no
2445    /// second HELLO.
2446    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447        if let Some(swap) = self
2448            .swaps
2449            .lock()
2450            .unwrap_or_else(|poisoned| poisoned.into_inner())
2451            .get_mut(module_id)
2452        {
2453            swap.candidate_admitted = true;
2454        }
2455    }
2456
2457    /// Test/support lookup for the current launch nonce of a supervised spawn.
2458    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459        self.spawn_nonces
2460            .lock()
2461            .unwrap_or_else(|poisoned| poisoned.into_inner())
2462            .get(module_id)
2463            .cloned()
2464    }
2465
2466    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2467    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468        self.reserved_nonces
2469            .lock()
2470            .unwrap_or_else(|poisoned| poisoned.into_inner())
2471            .get(module_id)
2472            .cloned()
2473            .flatten()
2474    }
2475
2476    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477        // Normally already marked before the process was spawned; marking here
2478        // too keeps `configured_ids` a superset of the roster for any caller
2479        // that inserts a module directly.
2480        self.mark_configured(module.module_id());
2481        let mut modules = self
2482            .modules
2483            .lock()
2484            .unwrap_or_else(|poisoned| poisoned.into_inner());
2485        modules.insert(module.module_id().to_string(), module)
2486    }
2487
2488    /// Record that the supervisor has taken on `module_id`. Called before the
2489    /// module's first process is spawned, so that by the time that process can
2490    /// register, [`Self::is_configured`] already answers true.
2491    fn mark_configured(&self, module_id: &str) {
2492        self.configured_ids
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner())
2495            .insert(module_id.to_string());
2496    }
2497
2498    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2499    /// before it was ever put on the roster. A module already on the roster
2500    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2501    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502        let modules = self
2503            .modules
2504            .lock()
2505            .unwrap_or_else(|poisoned| poisoned.into_inner());
2506        if !modules.contains_key(module_id) {
2507            self.configured_ids
2508                .lock()
2509                .unwrap_or_else(|poisoned| poisoned.into_inner())
2510                .remove(module_id);
2511        }
2512    }
2513
2514    /// Whether `module_id` is a module this daemon supervises: on the roster,
2515    /// or about to be (its process is being spawned right now).
2516    ///
2517    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2518    /// for scopes: a supervised module's process can register and sync before
2519    /// [`Self::get`] can return it, and in that window it is still a module
2520    /// that will sync, not one that never will.
2521    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522        self.configured_ids
2523            .lock()
2524            .unwrap_or_else(|poisoned| poisoned.into_inner())
2525            .contains(module_id)
2526    }
2527
2528    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529        let modules = self
2530            .modules
2531            .lock()
2532            .unwrap_or_else(|poisoned| poisoned.into_inner());
2533        modules.get(module_id).cloned()
2534    }
2535
2536    pub(crate) fn record_late_health_answer(
2537        &self,
2538        module_id: &str,
2539        latency_ms: u64,
2540    ) -> Result<bool, SuperviseError> {
2541        let Some(module) = self.get(module_id) else {
2542            return Ok(false);
2543        };
2544        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546            state.health.last_late_answer_latency_ms = Some(latency_ms);
2547            // A late answer is an answer: the module served the probe, just past
2548            // the deadline. Leaving the miss streak in place while logging
2549            // "proves the module is alive" is how a CPU-starved module that
2550            // answers every probe a few seconds late still marches to the
2551            // threshold and gets killed — the exact kill class `NoAnswer` is
2552            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2553            // is degradation, and degradation reports; it does not restart.
2554            state.health.consecutive_failures = 0;
2555        })?;
2556        Ok(true)
2557    }
2558
2559    /// Arm the one-shot marker for the module process that this caller
2560    /// deliberately initiated severance against. Generic connection teardown
2561    /// must not call this:
2562    /// a surviving process would otherwise retain an exemption for a later
2563    /// genuine crash.
2564    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565        let Some(module) = self.get(module_id) else {
2566            return Ok(false);
2567        };
2568        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570            return Ok(false);
2571        };
2572        drop(snapshot);
2573        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574    }
2575
2576    pub fn list(&self) -> Vec<SupervisedModule> {
2577        let modules = self
2578            .modules
2579            .lock()
2580            .unwrap_or_else(|poisoned| poisoned.into_inner());
2581        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583        modules
2584    }
2585
2586    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587        self.activity_images
2588            .lock()
2589            .unwrap_or_else(|p| p.into_inner())
2590            .remove(module_id);
2591        self.spawn_nonces
2592            .lock()
2593            .unwrap_or_else(|poisoned| poisoned.into_inner())
2594            .remove(module_id);
2595        self.close_swap(module_id);
2596        let mut reserved_nonces = self
2597            .reserved_nonces
2598            .lock()
2599            .unwrap_or_else(|poisoned| poisoned.into_inner());
2600        if reserved_nonces.contains_key(module_id) {
2601            // The old nonce must die with the removed process, but the exact-id
2602            // gate remains until an operator explicitly releases it.
2603            reserved_nonces.insert(module_id.to_string(), None);
2604        }
2605        drop(reserved_nonces);
2606        self.reserved_prefix_owners
2607            .lock()
2608            .unwrap_or_else(|poisoned| poisoned.into_inner())
2609            .retain(|_, owner| owner != module_id);
2610        let removed = self
2611            .modules
2612            .lock()
2613            .unwrap_or_else(|poisoned| poisoned.into_inner())
2614            .remove(module_id);
2615        self.configured_ids
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        removed
2620    }
2621
2622    /// Remember a module removed by a non-preview rescan so route.open can
2623    /// distinguish that intentional removal from an unknown id.
2624    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625        self.removal_tombstones
2626            .lock()
2627            .unwrap_or_else(|poisoned| poisoned.into_inner())
2628            .insert(module_id.to_string(), unix_ms_now());
2629    }
2630
2631    /// Return how long ago a rescan removed this module in milliseconds.
2632    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633        self.removal_tombstones
2634            .lock()
2635            .unwrap_or_else(|poisoned| poisoned.into_inner())
2636            .get(module_id)
2637            .copied()
2638            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639    }
2640
2641    /// Retire a reserved-id gate only after its module has left supervision.
2642    ///
2643    /// A retained gate has no live nonce (`None`), so releasing any other entry
2644    /// would weaken a currently configured or otherwise active reservation.
2645    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646        if self.get(module_id).is_some() {
2647            return false;
2648        }
2649        let mut reserved_nonces = self
2650            .reserved_nonces
2651            .lock()
2652            .unwrap_or_else(|poisoned| poisoned.into_inner());
2653        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654            return false;
2655        }
2656        reserved_nonces.remove(module_id);
2657        true
2658    }
2659
2660    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661        Arc::clone(&self.operation_lock)
2662    }
2663}
2664
2665/// Process supervisor for subc-owned singleton modules.
2666#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668    registry: Arc<Registry>,
2669    restart_policy: RestartPolicy,
2670    drain_timeout: Duration,
2671    connection_file_path: Option<PathBuf>,
2672    capture_logs_dir: Option<PathBuf>,
2673    forwarding: Option<Arc<ForwardingTable>>,
2674    process_liveness: Arc<SupervisorProcessLiveness>,
2675    supervisor_handle: Option<SupervisorHandle>,
2676    health: HealthConfig,
2677    daemon_start_clock: crate::clock::StartClock,
2678    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679    spawn_events: SpawnEventFeed,
2680    provenance_probe: ExecutableIdentityProbe,
2681    /// Every process spawned through this supervisor (and its clones) and not
2682    /// yet reaped, so daemon shutdown can end them.
2683    child_roster: ChildRoster,
2684    #[cfg(target_os = "linux")]
2685    cgroup_placement: Option<subc_cgroup::Placement>,
2686    #[cfg(test)]
2687    test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690/// A roster entry whose monitor and first process have not started yet.
2691/// Keeping the command receiver here lets discovery serve status immediately
2692/// while start/stop requests wait until bootstrap has published the connection file.
2693pub(crate) struct RegisteredConfiguredModule {
2694    spec: ModuleSpec,
2695    runtime: SupervisorRuntimeConfig,
2696    module: SupervisedModule,
2697    commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700/// Test-only hook run on the path that takes on a new module, right after its
2701/// first `spawn_child` returns (the process exists and could already be
2702/// registering) and before that process is handed to the module's supervise
2703/// loop and put on the roster. Lets a test observe what a fast child would see
2704/// in that window without racing a real one.
2705#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715        f.write_str("AfterFirstSpawnHook")
2716    }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721    fn run(&self, module_id: &str) {
2722        if let Some(hook) = &self.0 {
2723            hook(module_id);
2724        }
2725    }
2726}
2727
2728impl Supervisor {
2729    #[cfg(test)]
2730    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731        let supervisor = Self::new(registry, policy);
2732        #[cfg(target_os = "macos")]
2733        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734        supervisor
2735    }
2736    /// Verify the trampoline once when configured. Missing private OS support
2737    /// refuses every macOS launch by name but does not stop the daemon's control
2738    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2739    /// entry point; the library must not exec an arbitrary hosting program.
2740    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741        let path = path.into();
2742        #[cfg(target_os = "macos")]
2743        {
2744            let result = probe_privacy_trampoline(&path).map(|()| path);
2745            if let Err(cause) = &result {
2746                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747            }
2748            self.child_roster.set_privacy_trampoline(result);
2749        }
2750        #[cfg(not(target_os = "macos"))]
2751        let _ = path;
2752        self
2753    }
2754    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2755    /// capability-requirement log to read.
2756    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2757        self.child_roster.shutdown_flag()
2758    }
2759
2760    /// The first step of an announced daemon shutdown, before the notice and
2761    /// before any connection is closed.
2762    ///
2763    /// Sets the daemon-shutdown flag first: from here on no module is
2764    /// respawned (crash restart, operator restart, or swap), and every child
2765    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2766    /// the module exits on the EOF this shutdown gives it or is signalled by a
2767    /// service manager that kills the whole cgroup. Then writes the journal's
2768    /// shutdown marker, which records the instant and closes this daemon
2769    /// incarnation's stretch of the journal.
2770    #[cfg(unix)]
2771    pub(crate) fn begin_daemon_shutdown(&self) {
2772        self.child_roster.close();
2773        if let Some(journal) = &self.terminal_journal {
2774            journal.stamp_shutdown();
2775        }
2776    }
2777
2778    /// Announce a cut while established connections can still carry replies.
2779    /// These budgets promise notice and a bounded wait, not child completion;
2780    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2781    #[cfg(unix)]
2782    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2783        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2784        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2785        let Some(forwarding) = &self.forwarding else {
2786            return Ok(());
2787        };
2788        let modules = forwarding
2789            .begin_daemon_drain()
2790            .map_err(SuperviseError::Forwarding)?;
2791        let deadline_ms =
2792            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2793        let mut notices = tokio::task::JoinSet::new();
2794        let mut drains = Vec::new();
2795        for (module_id, endpoint) in modules {
2796            let Some(target) = forwarding
2797                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2798                .map_err(SuperviseError::Forwarding)?
2799            else {
2800                continue;
2801            };
2802            let routes = forwarding
2803                .endpoint_routes(target.endpoint)
2804                .map_err(SuperviseError::Forwarding)?;
2805            // Restart allows deployed consumers to reopen after the new daemon
2806            // appears. The wire reason stays `restart`; what tells a daemon cut
2807            // apart from a module restart afterwards is the terminal record
2808            // itself, whose disposition is `daemon_shutdown` for every exit
2809            // observed once `begin_daemon_shutdown` has run.
2810            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2811                reason: RouteCloseReason::Restart,
2812                deadline_ms,
2813            })
2814            .expect("module draining serializes");
2815            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2816            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2817            for route in routes {
2818                let client = route.goodbye_target;
2819                if let Some((_, channels)) = clients
2820                    .iter_mut()
2821                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2822                {
2823                    channels.push(client.channel);
2824                } else {
2825                    let channel = client.channel;
2826                    clients.push((client, vec![channel]));
2827                }
2828            }
2829            for (client, mut channels) in clients {
2830                channels.sort_unstable();
2831                channels.dedup();
2832                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2833                    module_id: module_id.clone(),
2834                    channels,
2835                    reason: RouteCloseReason::Restart,
2836                })
2837                .expect("route closing serializes");
2838                recipients.push((client.sink, client.negotiated_ver, closing));
2839            }
2840            for (sink, version, body) in recipients {
2841                notices.spawn(async move {
2842                    let frame = Frame::build_with_version(
2843                        version,
2844                        FrameType::Push,
2845                        control_flags(),
2846                        0,
2847                        0,
2848                        0,
2849                        body,
2850                    )
2851                    .expect("bounded lifecycle notice frame builds");
2852                    sink.send_flushed(frame).await
2853                });
2854            }
2855            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2856            drains.push((module_id, target.endpoint, gauges));
2857        }
2858        // A quiet forwarding table is not proof that queued notices reached the
2859        // socket. Wait for writer flush acknowledgements before testing quiescence.
2860        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2861        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2862            if !matches!(result, Ok(Ok(()))) {
2863                warn!(?result, "daemon shutdown notice delivery failed");
2864            }
2865        }
2866        notices.abort_all();
2867        let deadline = Instant::now() + DRAIN_BUDGET;
2868        let mut waits = tokio::task::JoinSet::new();
2869        for (module_id, endpoint, gauges) in drains {
2870            let forwarding = Arc::clone(forwarding);
2871            let mut runtime = self.runtime_config();
2872            runtime.health.cadence = Duration::from_millis(100);
2873            waits.spawn(async move {
2874                let wait_result = wait_for_forwarding_quiescence(
2875                    &forwarding,
2876                    &module_id,
2877                    &runtime,
2878                    endpoint,
2879                    deadline,
2880                    &gauges,
2881                    DrainScope::Endpoint(endpoint),
2882                    &mut DrainGaugeTrace::new(&gauges),
2883                )
2884                .await;
2885                // Store the measurement before the task returns: quiescence
2886                // may be reached just as the deadline stops the join loop from
2887                // collecting results. Unfinished or failed waits report false.
2888                forwarding
2889                    .record_daemon_drain_result(
2890                        endpoint,
2891                        drained_after_quiescence_wait(&wait_result),
2892                    )
2893                    .map_err(SuperviseError::Forwarding)?;
2894                wait_result
2895            });
2896        }
2897        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2898            if !matches!(result, Ok(Ok(true))) {
2899                warn!(?result, "daemon shutdown drain did not reach quiescence");
2900            }
2901        }
2902        Ok(())
2903    }
2904
2905    /// The last step of an announced daemon shutdown, after the notice and the
2906    /// drain: send every registered module a module GOODBYE, the same planned
2907    /// stop signal `ck module stop` gives, then close every connection so each
2908    /// subc module sees EOF and starts its own teardown, then end every
2909    /// supervised child that has not exited
2910    /// by its own deadline (its drain budget, capped). Modules lead their own
2911    /// process groups, so a
2912    /// service manager's group kill no longer reaches them; without this a
2913    /// child that does not stop on EOF (every `protocol: "none"` child, which
2914    /// has no connection) would outlive the daemon. Every wait is bounded (see
2915    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2916    #[cfg(unix)]
2917    pub(crate) async fn end_children_for_daemon_shutdown(
2918        &self,
2919        already_escalated: bool,
2920        escalate: impl std::future::Future<Output = ()>,
2921    ) {
2922        tokio::pin!(escalate);
2923        let mut escalated = already_escalated;
2924        if let Some(forwarding) = &self.forwarding {
2925            let reason = CloseReason::new(
2926                "daemon_shutdown",
2927                "the daemon is exiting after its shutdown notice and drain",
2928            );
2929            if escalated {
2930                // The operator asked to stop waiting: queue the GOODBYEs but
2931                // do not wait for them to be written.
2932                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2933            } else {
2934                tokio::select! {
2935                    biased;
2936                    _ = escalate.as_mut() => {
2937                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2938                        escalated = true;
2939                    }
2940                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2941                }
2942            }
2943            let closed = forwarding.close_all_connections(&reason);
2944            debug!(closed, "closed established connections for daemon shutdown");
2945        }
2946        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2947        // already completed and must not be polled again; the child shutdown
2948        // wait is told it is escalated and gets a future that never fires.
2949        let escalated_here = escalated && !already_escalated;
2950        let remaining_escalate = async move {
2951            if escalated_here {
2952                std::future::pending::<()>().await;
2953            } else {
2954                escalate.await;
2955            }
2956        };
2957        crate::child_roster::end_children_for_daemon_shutdown(
2958            &self.child_roster,
2959            escalated,
2960            remaining_escalate,
2961        )
2962        .await;
2963    }
2964
2965    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2966        Self {
2967            registry,
2968            restart_policy,
2969            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2970            connection_file_path: None,
2971            capture_logs_dir: None,
2972            forwarding: None,
2973            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2974            supervisor_handle: None,
2975            health: HealthConfig::default(),
2976            daemon_start_clock: crate::clock::StartClock::capture(),
2977            terminal_journal: None,
2978            spawn_events: SpawnEventFeed::default(),
2979            provenance_probe: ExecutableIdentityProbe::default(),
2980            child_roster: ChildRoster::default(),
2981            #[cfg(target_os = "linux")]
2982            cgroup_placement: None,
2983            #[cfg(test)]
2984            test_after_first_spawn: AfterFirstSpawnHook::default(),
2985        }
2986    }
2987
2988    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2989        self.drain_timeout = drain_timeout;
2990        self
2991    }
2992
2993    /// The drain budget a module gets when its config entry sets none.
2994    pub(crate) fn default_drain_timeout(&self) -> Duration {
2995        self.drain_timeout
2996    }
2997
2998    pub fn with_process_liveness(
2999        mut self,
3000        process_liveness: Arc<SupervisorProcessLiveness>,
3001    ) -> Self {
3002        self.process_liveness = process_liveness;
3003        self
3004    }
3005
3006    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3007        self.connection_file_path = Some(connection_file_path.into());
3008        self
3009    }
3010
3011    /// Enables daemon-owned capture files for supervised stdout and stderr.
3012    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3013        self.capture_logs_dir = Some(logs_dir.into());
3014        self
3015    }
3016
3017    /// Names this daemon lifetime in spawn events, independently of whether a
3018    /// terminal journal is configured.
3019    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3020        // A millisecond start stamp can repeat after clock rollback or a rapid
3021        // restart. Use the connection file's random daemon_id instead: it already
3022        // identifies this daemon lifetime independently of the wall clock.
3023        self.spawn_events.configure_incarnation(daemon_incarnation);
3024        self
3025    }
3026
3027    /// Enables best-effort history shared by every supervised module. Without
3028    /// it, terminal history is kept only in each module's in-memory ring.
3029    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3030        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3031        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3032            path,
3033            daemon_incarnation,
3034        )));
3035        this
3036    }
3037
3038    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3039        self.forwarding = Some(forwarding);
3040        self
3041    }
3042
3043    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3044        self.spawn_events = supervisor_handle.spawn_events.clone();
3045        self.supervisor_handle = Some(supervisor_handle);
3046        self
3047    }
3048
3049    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3050        self.health = health;
3051        self
3052    }
3053
3054    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3055    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3056    /// record is kept.
3057    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3058        self.child_roster.record_to(path.into());
3059        self
3060    }
3061
3062    #[cfg(target_os = "linux")]
3063    pub fn with_cgroup_placement(
3064        mut self,
3065        cgroup_placement: Option<subc_cgroup::Placement>,
3066    ) -> Self {
3067        self.cgroup_placement = cgroup_placement;
3068        self
3069    }
3070
3071    /// Spawn `spec.program` and start monitoring it.
3072    ///
3073    /// The child is expected to parse `--subc <connection-file-path>`, read the
3074    /// TCP+key connection file, authenticate to the already-running listener, and
3075    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3076    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3077        validate_spec(&spec)?;
3078        self.establish_identity(&spec);
3079
3080        let runtime = self.runtime_config();
3081        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3082        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3083        let spawned = spawn_child(
3084            &spec,
3085            runtime.connection_file_path.as_deref(),
3086            self.supervisor_handle.as_ref(),
3087            &runtime.stderr_ring,
3088            runtime.capture_logs_dir.as_deref(),
3089            &runtime.child_roster,
3090            #[cfg(target_os = "linux")]
3091            runtime.cgroup_placement.as_ref(),
3092        );
3093        #[cfg(test)]
3094        self.test_after_first_spawn.run(&spec.module_id);
3095        let child = match spawned {
3096            Ok(child) => child,
3097            Err(err) => {
3098                // Unlike the configured paths, a failed `spawn` leaves nothing
3099                // on the roster, so the module must not stay marked configured.
3100                self.abandon_unrostered(&spec.module_id);
3101                return Err(err);
3102            }
3103        };
3104        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3105        if let Some(spawn) = activity_spawn {
3106            spawn.complete(&snapshot);
3107        }
3108
3109        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3110    }
3111
3112    /// Make `spec`'s module count as configured, with its identity gates
3113    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3114    /// exists.
3115    ///
3116    /// Every path that takes on a new module calls this before `spawn_child`.
3117    /// The order is the point: the child can connect, register, sync its
3118    /// scopes and ask about them as soon as it is spawned, and the module is
3119    /// only put on the roster after `spawn_child` returns. Were the mark set
3120    /// with the roster entry, a fast child would see its own owner reported
3121    /// as not configured, and a scoped `route.open` in that window would be
3122    /// refused as terminal `scope_not_live` ("will never sync") instead of
3123    /// retryable `scope_not_synced`.
3124    fn establish_identity(&self, spec: &ModuleSpec) {
3125        if let Some(supervisor_handle) = &self.supervisor_handle {
3126            supervisor_handle.apply_identity_configuration(spec);
3127            supervisor_handle.mark_configured(&spec.module_id);
3128        }
3129    }
3130
3131    /// Take back [`Self::establish_identity`]'s configured mark when the
3132    /// module will not be put on the roster after all.
3133    fn abandon_unrostered(&self, module_id: &str) {
3134        if let Some(supervisor_handle) = &self.supervisor_handle {
3135            supervisor_handle.unmark_configured_unless_rostered(module_id);
3136        }
3137    }
3138
3139    /// Record a freshly spawned first process as running. On failure the
3140    /// module never reaches the roster, so its configured mark is taken back.
3141    fn mark_first_process_running(
3142        &self,
3143        spec: &ModuleSpec,
3144        runtime: &SupervisorRuntimeConfig,
3145        snapshot: &SharedSnapshot,
3146        child: &SupervisedChild,
3147    ) -> Result<(), SuperviseError> {
3148        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3149            self.abandon_unrostered(&spec.module_id);
3150            return Err(err);
3151        }
3152        self.process_liveness
3153            .track(spec.module_id.clone(), Arc::clone(snapshot));
3154        Ok(())
3155    }
3156
3157    /// Start supervising a module declared in daemon configuration.
3158    ///
3159    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3160    /// failures in the supervisor handle so operator-facing `supervisor.list`
3161    /// reflects every configured module while daemon startup continues.
3162    pub fn supervise_configured(
3163        &self,
3164        spec: ModuleSpec,
3165        enabled: bool,
3166    ) -> Result<SupervisedModule, SuperviseError> {
3167        self.supervise_configured_requested(
3168            spec,
3169            enabled,
3170            self.health.clone(),
3171            None,
3172            self.restart_policy,
3173            None,
3174        )
3175    }
3176
3177    /// Supervise a configured module with its own health, drain, and crash
3178    /// budget. The restart policy is per-module because the config file is:
3179    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3180    /// module that is expensive to restart should not be forced onto the same
3181    /// budget as one that is cheap.
3182    pub fn supervise_configured_with_health(
3183        &self,
3184        spec: ModuleSpec,
3185        enabled: bool,
3186        health: HealthConfig,
3187        drain_timeout_ms: Option<u64>,
3188        restart_policy: RestartPolicy,
3189    ) -> Result<SupervisedModule, SuperviseError> {
3190        self.supervise_configured_requested(
3191            spec,
3192            enabled,
3193            health,
3194            drain_timeout_ms,
3195            restart_policy,
3196            None,
3197        )
3198    }
3199
3200    pub(crate) fn supervise_configured_requested(
3201        &self,
3202        spec: ModuleSpec,
3203        enabled: bool,
3204        health: HealthConfig,
3205        drain_timeout_ms: Option<u64>,
3206        restart_policy: RestartPolicy,
3207        requester: Option<Principal>,
3208    ) -> Result<SupervisedModule, SuperviseError> {
3209        let registered = self.register_configured_with_health(
3210            spec,
3211            enabled,
3212            health,
3213            drain_timeout_ms,
3214            restart_policy,
3215        )?;
3216        self.start_registered_configured(registered, requester)
3217    }
3218
3219    /// Record a configured module as starting or disabled, without launching a
3220    /// process or running its monitor. Bootstrap registers the entire roster
3221    /// before publishing discovery, then activates it once children can read
3222    /// the connection file passed in their `--subc` argument.
3223    pub(crate) fn register_configured_with_health(
3224        &self,
3225        spec: ModuleSpec,
3226        enabled: bool,
3227        health: HealthConfig,
3228        drain_timeout_ms: Option<u64>,
3229        restart_policy: RestartPolicy,
3230    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3231        validate_spec(&spec)?;
3232        self.establish_identity(&spec);
3233
3234        let mut runtime = self.runtime_config();
3235        runtime.health = health;
3236        runtime.restart_policy = restart_policy;
3237        if let Some(ms) = drain_timeout_ms {
3238            runtime.drain_timeout = Duration::from_millis(ms);
3239            *runtime
3240                .effective_drain_timeout
3241                .lock()
3242                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3243        }
3244        let snapshot = if enabled {
3245            SupervisorSnapshot::starting()
3246        } else {
3247            SupervisorSnapshot::disabled()
3248        };
3249        let snapshot = {
3250            let mut snapshot = snapshot;
3251            if enabled {
3252                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3253                {
3254                    snapshot.waiting_for_previous_pid = Some(pid);
3255                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3256                }
3257            }
3258            snapshot
3259        };
3260        let (module, commands) =
3261            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3262        Ok(RegisteredConfiguredModule {
3263            spec,
3264            runtime,
3265            module,
3266            commands,
3267        })
3268    }
3269
3270    /// Activate a previously registered module. Queued control requests are
3271    /// served by the monitor after the initial configured spawn has settled.
3272    pub(crate) fn start_registered_configured(
3273        &self,
3274        registered: RegisteredConfiguredModule,
3275        requester: Option<Principal>,
3276    ) -> Result<SupervisedModule, SuperviseError> {
3277        let RegisteredConfiguredModule {
3278            spec,
3279            mut runtime,
3280            module,
3281            commands,
3282        } = registered;
3283        runtime.start_requester = requester;
3284        let snapshot = &module.inner.snapshot;
3285        let should_spawn = {
3286            let state = lock_snapshot(snapshot)?;
3287            state.enabled && state.waiting_for_previous_pid.is_none()
3288        };
3289        let child = if should_spawn {
3290            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3291            let spawned = spawn_child(
3292                &spec,
3293                runtime.connection_file_path.as_deref(),
3294                self.supervisor_handle.as_ref(),
3295                &runtime.stderr_ring,
3296                runtime.capture_logs_dir.as_deref(),
3297                &runtime.child_roster,
3298                #[cfg(target_os = "linux")]
3299                runtime.cgroup_placement.as_ref(),
3300            );
3301            #[cfg(test)]
3302            self.test_after_first_spawn.run(&spec.module_id);
3303            match spawned {
3304                Ok(child) => {
3305                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3306                    if let Some(spawn) = activity_spawn {
3307                        spawn.complete(snapshot);
3308                    }
3309                    Some(child)
3310                }
3311                Err(err) => {
3312                    if runtime.health.critical {
3313                        error!(
3314                            module_id = %spec.module_id,
3315                            program = %spec.program.display(),
3316                            error = %err,
3317                            "critical configured module failed to spawn; marking failed and alerting"
3318                        );
3319                    } else {
3320                        error!(
3321                            module_id = %spec.module_id,
3322                            program = %spec.program.display(),
3323                            error = %err,
3324                            "configured module failed to spawn; marking failed and continuing"
3325                        );
3326                    }
3327                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3328                        state.state = ModuleState::Failed;
3329                        clear_current_process_facts(state);
3330                    })?;
3331                    None
3332                }
3333            }
3334        } else {
3335            None
3336        };
3337        self.start_monitor(spec, runtime, &module, child, commands);
3338        Ok(module)
3339    }
3340
3341    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3342        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3343        SupervisorRuntimeConfig {
3344            start_requester: None,
3345            pending_activity_restart: false,
3346            provenance_probe: self.provenance_probe.clone(),
3347            scheduled_respawn: Arc::default(),
3348            deferred_reload_reply: Arc::default(),
3349            restart_policy: self.restart_policy,
3350            drain_timeout: self.drain_timeout,
3351            // Shared with this module's roster copy: daemon shutdown waits on
3352            // each child for the module's own drain budget, as resolved now.
3353            child_roster: self
3354                .child_roster
3355                .for_module(Arc::clone(&effective_drain_timeout)),
3356            effective_drain_timeout,
3357            default_drain_timeout: self.drain_timeout,
3358            health: self.health.clone(),
3359            connection_file_path: self.connection_file_path.clone(),
3360            capture_logs_dir: self.capture_logs_dir.clone(),
3361            forwarding: self.forwarding.clone(),
3362            supervisor_handle: self.supervisor_handle.clone(),
3363            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3364            terminal_ring: Arc::new(Mutex::new(
3365                TerminalRing::new(
3366                    TerminalRingConfig::default(),
3367                    self.daemon_start_clock.started_at_ms(),
3368                )
3369                .with_start_clock(self.daemon_start_clock)
3370                .with_journal(self.terminal_journal.clone())
3371                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3372            )),
3373            spawn_events: self.spawn_events.clone(),
3374            #[cfg(target_os = "linux")]
3375            cgroup_placement: self.cgroup_placement.clone(),
3376            #[cfg(test)]
3377            test_seed_stale_facts_before_enable_spawn: false,
3378            #[cfg(test)]
3379            test_reload_exit_record_gate: None,
3380        }
3381    }
3382
3383    fn supervised_module(
3384        &self,
3385        spec: ModuleSpec,
3386        runtime: SupervisorRuntimeConfig,
3387        snapshot: SharedSnapshot,
3388        child: Option<SupervisedChild>,
3389    ) -> SupervisedModule {
3390        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3391        self.start_monitor(spec, runtime, &module, child, commands);
3392        module
3393    }
3394
3395    fn register_module(
3396        &self,
3397        spec: &ModuleSpec,
3398        runtime: &SupervisorRuntimeConfig,
3399        snapshot: SharedSnapshot,
3400    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3401        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3402            spec: spec.clone(),
3403            health: runtime.health.clone(),
3404        }));
3405        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3406        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3407        // The module's OWN policy, which may be its per-module config rather than
3408        // the supervisor-wide one; status must report the budget the supervise
3409        // loop actually enforces.
3410        let restart_policy = runtime.restart_policy;
3411        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3412        let (tx, rx) = mpsc::channel(4);
3413        let module_id = spec.module_id.clone();
3414        let module = SupervisedModule {
3415            inner: Arc::new(SupervisedModuleInner {
3416                module_id: module_id.clone(),
3417                registry: Arc::clone(&self.registry),
3418                snapshot,
3419                configuration,
3420                stderr_ring,
3421                terminal_ring,
3422                commands: tx,
3423                monitor: Mutex::new(None),
3424                restart_policy,
3425                effective_drain_timeout,
3426                provenance_probe: self.provenance_probe.clone(),
3427            }),
3428        };
3429        // Identity gates and the configured mark must precede the roster entry,
3430        // so a client never sees a module without its admission policy.
3431        if let Some(supervisor_handle) = &self.supervisor_handle {
3432            supervisor_handle.insert(module.clone());
3433        }
3434        (module, rx)
3435    }
3436
3437    fn start_monitor(
3438        &self,
3439        spec: ModuleSpec,
3440        runtime: SupervisorRuntimeConfig,
3441        module: &SupervisedModule,
3442        child: Option<SupervisedChild>,
3443        commands: mpsc::Receiver<SupervisorCommand>,
3444    ) {
3445        let monitor = tokio::spawn(supervise_loop(
3446            spec,
3447            runtime,
3448            Arc::clone(&self.registry),
3449            Arc::clone(&self.process_liveness),
3450            Arc::clone(&module.inner.snapshot),
3451            child,
3452            commands,
3453        ));
3454        *module
3455            .inner
3456            .monitor
3457            .lock()
3458            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3459    }
3460}
3461
3462impl Default for Supervisor {
3463    fn default() -> Self {
3464        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3465    }
3466}
3467
3468/// Handle to one supervised child process.
3469#[derive(Clone)]
3470pub struct SupervisedModule {
3471    inner: Arc<SupervisedModuleInner>,
3472}
3473
3474struct SupervisedModuleInner {
3475    module_id: String,
3476    registry: Arc<Registry>,
3477    snapshot: SharedSnapshot,
3478    configuration: Arc<Mutex<SupervisedConfiguration>>,
3479    stderr_ring: Arc<Mutex<StderrRing>>,
3480    terminal_ring: Arc<Mutex<TerminalRing>>,
3481    commands: mpsc::Sender<SupervisorCommand>,
3482    monitor: Mutex<Option<JoinHandle<()>>>,
3483    /// Copied from the supervisor's runtime config at spawn so `status()` can
3484    /// report the restart budget without reaching back into the supervisor. The
3485    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3486    restart_policy: RestartPolicy,
3487    effective_drain_timeout: Arc<Mutex<Duration>>,
3488    provenance_probe: ExecutableIdentityProbe,
3489}
3490
3491impl fmt::Debug for SupervisedModule {
3492    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3493        f.debug_struct("SupervisedModule")
3494            .field("module_id", &self.inner.module_id)
3495            .field("status", &self.status())
3496            .finish_non_exhaustive()
3497    }
3498}
3499
3500impl SupervisedModule {
3501    pub fn module_id(&self) -> &str {
3502        &self.inner.module_id
3503    }
3504
3505    /// Test-only: put one probe miss on the streak, the way
3506    /// `handle_health_probe_failure` does, so tests can assert what a later
3507    /// event does to the streak without driving the whole probe loop.
3508    #[cfg(test)]
3509    pub(crate) fn record_health_probe_failure_for_test(
3510        &self,
3511        detail: &str,
3512    ) -> Result<(), SuperviseError> {
3513        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3514            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3515            state.health.detail = Some(detail.to_string());
3516        })
3517    }
3518
3519    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3520        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3521    }
3522
3523    pub(crate) fn registration_end_reason(
3524        &self,
3525    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3526        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3527        Ok(match snapshot.state {
3528            ModuleState::Draining if snapshot.draining_to_replace => {
3529                Some(RegistrationEndReason::SupervisorRestart)
3530            }
3531            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3532            ModuleState::Restarting if snapshot.draining_to_replace => {
3533                Some(RegistrationEndReason::SupervisorRestart)
3534            }
3535            _ => None,
3536        })
3537    }
3538
3539    /// The module's retained stderr, newest lines last.
3540    ///
3541    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3542    /// module, `supervisor.list` renders every module, and putting it in the
3543    /// shared snapshot would make each status read carry a payload almost nobody
3544    /// asked for. Callers that want the text ask for it.
3545    pub fn stderr_tail(
3546        &self,
3547        max_lines: Option<usize>,
3548        max_bytes: Option<usize>,
3549    ) -> StderrTailSnapshot {
3550        self.inner
3551            .stderr_ring
3552            .lock()
3553            .unwrap_or_else(|poisoned| poisoned.into_inner())
3554            .snapshot(max_lines, max_bytes)
3555    }
3556
3557    /// The module's bounded terminal history, oldest retained exit first.
3558    ///
3559    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3560    /// daemon whose in-memory history was necessarily reset.
3561    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3562        self.inner
3563            .terminal_ring
3564            .lock()
3565            .unwrap_or_else(|poisoned| poisoned.into_inner())
3566            .snapshot()
3567    }
3568
3569    /// Retained observations from the current ring and all journal generations.
3570    ///
3571    /// Blocking: this reads the journal files. Async callers use
3572    /// [`Self::read_durable_terminal_history`].
3573    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3574        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3575    }
3576
3577    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3578    /// read (up to every retained generation) never occupies a runtime worker.
3579    /// Fails only if the blocking task could not finish (runtime shutdown or a
3580    /// panic in the read).
3581    pub(crate) async fn read_durable_terminal_history(
3582        &self,
3583    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3584        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3585        let module_id = self.inner.module_id.clone();
3586        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3587            .await
3588    }
3589
3590    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3591        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3592            .map(|(status, _)| status)
3593    }
3594
3595    pub(crate) fn record_deliberate_severance(
3596        &self,
3597        identity: ProcessIdentity,
3598    ) -> Result<bool, SuperviseError> {
3599        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3600        if snapshot.pid != Some(identity.pid)
3601            || snapshot.process_start_time != Some(identity.start_time)
3602        {
3603            return Ok(false);
3604        }
3605        snapshot.deliberate_severance = Some(identity);
3606        Ok(true)
3607    }
3608
3609    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3610    ///
3611    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3612    /// does not produce reader-observability logs.
3613    pub(crate) fn status_for_control(
3614        &self,
3615        caller: &'static str,
3616    ) -> Result<ModuleStatus, SuperviseError> {
3617        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3618            .map(|(status, _)| status)
3619    }
3620
3621    fn status_with_snapshot_lock(
3622        &self,
3623        snapshot: &SharedSnapshot,
3624        caller: Option<&'static str>,
3625    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3626        let mut guard = match caller {
3627            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3628            None => lock_snapshot(snapshot)?,
3629        };
3630        // Read the budget through the pruning path so a reader sees the same
3631        // in-window count the restart decision would use, not a stale total.
3632        let restart_count =
3633            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3634        let snapshot = guard.clone();
3635        drop(guard);
3636        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3637            SuperviseError::StatePoisoned {
3638                module_id: Some(self.inner.module_id.clone()),
3639            }
3640        })?;
3641        let registration_active = self
3642            .inner
3643            .registry
3644            .get_module(&self.inner.module_id)
3645            .map_err(SuperviseError::Registry)?
3646            .is_some();
3647        let protocol = snapshot
3648            .spawned_protocol
3649            .unwrap_or(self.declared_protocol()?);
3650        let running_process =
3651            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3652        // Registration is the difference between the two protocols and the only
3653        // one: a subc module that has not registered cannot serve a request even
3654        // though its process is up, and a `none` module never registers at all,
3655        // so requiring it there would pin `live` to false for the whole life of
3656        // a perfectly healthy process.
3657        let live = match protocol {
3658            ModuleProtocol::Subc => running_process && registration_active,
3659            ModuleProtocol::None => running_process,
3660        };
3661
3662        Ok((
3663            ModuleStatus {
3664                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3665                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3666                module_id: self.inner.module_id.clone(),
3667                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3668                state: snapshot.state,
3669                enabled: snapshot.enabled,
3670                process_alive: snapshot.process_alive,
3671                registration_active,
3672                protocol,
3673                live,
3674                restart_count,
3675                lifetime_restarts: snapshot.lifetime_restarts,
3676                spawn_generation: snapshot.spawn_generation,
3677                max_restarts: self.inner.restart_policy.max_restarts,
3678                restart_window: self.inner.restart_policy.window,
3679                drain_timeout,
3680                restart_backoff: self.inner.restart_policy.backoff,
3681                restart_max_backoff: self.inner.restart_policy.max_backoff,
3682                pid: snapshot.reported_pid(),
3683                spawned_at_ms: snapshot.spawned_at_ms,
3684                spawned_from: snapshot.spawned_from,
3685                launch_spec_digest: snapshot.launch_spec_digest,
3686                process_start_time: snapshot.process_start_time,
3687                last_exit: snapshot.last_exit,
3688                health: snapshot.health,
3689            },
3690            snapshot.spawned_file_identity.clone(),
3691        ))
3692    }
3693
3694    #[cfg(test)]
3695    pub(crate) fn hold_snapshot_for_test(
3696        &self,
3697        acquired: std::sync::mpsc::Sender<()>,
3698        hold: Duration,
3699    ) -> std::thread::JoinHandle<()> {
3700        let snapshot = Arc::clone(&self.inner.snapshot);
3701        std::thread::spawn(move || {
3702            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3703            acquired
3704                .send(())
3705                .expect("test receiver waits for snapshot lock");
3706            std::thread::sleep(hold);
3707        })
3708    }
3709
3710    /// The status and the running-image check for `supervisor.provenance`,
3711    /// taken from one status read. The exec acknowledgement can land between
3712    /// two separate reads, and the reply would then pair "no pid yet" with an
3713    /// image observed after the module started, which describes no single
3714    /// moment.
3715    pub(crate) async fn status_and_running_image_agreement(
3716        &self,
3717    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3718        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3719        let image = self
3720            .inner
3721            .provenance_probe
3722            .observe(
3723                status.pid,
3724                status.spawned_from.as_deref(),
3725                identity,
3726                status.process_start_time,
3727            )
3728            .await;
3729        Ok((status, image))
3730    }
3731
3732    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3733        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3734            Ok(snapshot) => snapshot.clone(),
3735            Err(_) => {
3736                return subc_control::RunningImageAgreement::Unavailable {
3737                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3738                };
3739            }
3740        };
3741        self.inner
3742            .provenance_probe
3743            .observe(
3744                snapshot.reported_pid(),
3745                snapshot.spawned_from.as_deref(),
3746                snapshot.spawned_file_identity.clone(),
3747                snapshot.process_start_time,
3748            )
3749            .await
3750    }
3751
3752    /// Memory and CPU time of the module's current process, read now. Only the
3753    /// process the supervisor spawned is read, not processes it has started.
3754    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3755        #[cfg(windows)]
3756        {
3757            let image = lock_snapshot(&self.inner.snapshot)
3758                .ok()
3759                .and_then(|snapshot| {
3760                    snapshot
3761                        .spawned_file_identity
3762                        .as_ref()
3763                        .and_then(|identity| identity.windows_image.clone())
3764                });
3765            match image {
3766                Some(image) => {
3767                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3768                }
3769                None => subc_control::ChildResourceUsage::Unavailable {
3770                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3771                },
3772            }
3773        }
3774        #[cfg(not(windows))]
3775        {
3776            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3777                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3778                Err(_) => {
3779                    return subc_control::ChildResourceUsage::Unavailable {
3780                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3781                    }
3782                }
3783            };
3784            crate::child_resources::read(pid, start_time)
3785        }
3786    }
3787
3788    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3789        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3790        Ok(match snapshot.state {
3791            ModuleState::Restarting => true,
3792            ModuleState::Failed | ModuleState::Disabled => false,
3793            _ if snapshot.enabled && snapshot.respawn_pending => true,
3794            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3795        })
3796    }
3797
3798    #[cfg(test)]
3799    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3800        self.is_warming_with_snapshot_lock(None)
3801    }
3802
3803    pub(crate) fn is_warming_for_control(
3804        &self,
3805        caller: &'static str,
3806    ) -> Result<bool, SuperviseError> {
3807        self.is_warming_with_snapshot_lock(Some(caller))
3808    }
3809
3810    fn is_warming_with_snapshot_lock(
3811        &self,
3812        caller: Option<&'static str>,
3813    ) -> Result<bool, SuperviseError> {
3814        let snapshot = match caller {
3815            Some(caller) => {
3816                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3817            }
3818            None => lock_snapshot(&self.inner.snapshot)?,
3819        }
3820        .clone();
3821        Ok(matches!(
3822            snapshot.state,
3823            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3824        ))
3825    }
3826
3827    /// Drain the module and stop monitoring it.
3828    pub async fn drain(&self) -> Result<(), SuperviseError> {
3829        self.stop().await
3830    }
3831
3832    pub(crate) async fn retire_requested(
3833        &self,
3834        requester: Option<Principal>,
3835    ) -> Result<(), SuperviseError> {
3836        match self.state()? {
3837            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3838            ModuleState::Starting
3839            | ModuleState::Running
3840            | ModuleState::Unresponsive
3841            | ModuleState::Restarting
3842            | ModuleState::Draining
3843            | ModuleState::Disabled => {}
3844        }
3845
3846        let (reply_tx, reply_rx) = oneshot::channel();
3847        self.inner
3848            .commands
3849            .send(SupervisorCommand::Retire {
3850                requester,
3851                reply: reply_tx,
3852            })
3853            .await
3854            .map_err(|_| SuperviseError::CommandClosed {
3855                module_id: self.inner.module_id.clone(),
3856            })?;
3857        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3858            module_id: self.inner.module_id.clone(),
3859        })?
3860    }
3861
3862    pub async fn stop(&self) -> Result<(), SuperviseError> {
3863        self.stop_requested(None).await
3864    }
3865
3866    pub(crate) async fn stop_requested(
3867        &self,
3868        requester: Option<Principal>,
3869    ) -> Result<(), SuperviseError> {
3870        match self.state()? {
3871            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3872            ModuleState::Starting
3873            | ModuleState::Running
3874            | ModuleState::Unresponsive
3875            | ModuleState::Restarting
3876            | ModuleState::Draining
3877            | ModuleState::Disabled => {}
3878        }
3879
3880        let (reply_tx, reply_rx) = oneshot::channel();
3881        self.inner
3882            .commands
3883            .send(SupervisorCommand::Drain {
3884                requester,
3885                reply: reply_tx,
3886            })
3887            .await
3888            .map_err(|_| SuperviseError::CommandClosed {
3889                module_id: self.inner.module_id.clone(),
3890            })?;
3891        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3892            module_id: self.inner.module_id.clone(),
3893        })?
3894    }
3895
3896    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3897        self.restart_requested(drain_timeout_ms, None).await
3898    }
3899
3900    pub(crate) async fn restart_requested(
3901        &self,
3902        drain_timeout_ms: Option<u64>,
3903        requester: Option<Principal>,
3904    ) -> Result<(), SuperviseError> {
3905        let queued_at = Instant::now();
3906        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3907        let (reply_tx, reply_rx) = oneshot::channel();
3908        self.inner
3909            .commands
3910            .send(SupervisorCommand::Restart {
3911                requester,
3912                drain_timeout_ms,
3913                received_at_generation,
3914                queued_at,
3915                reply: reply_tx,
3916            })
3917            .await
3918            .map_err(|_| SuperviseError::CommandClosed {
3919                module_id: self.inner.module_id.clone(),
3920            })?;
3921        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3922            module_id: self.inner.module_id.clone(),
3923        })?
3924    }
3925
3926    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3927    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3928    /// process then drains in the background of the supervise loop) or has
3929    /// failed, leaving the old process serving.
3930    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3931        self.swap_requested(ready_timeout, None).await
3932    }
3933
3934    pub(crate) async fn swap_requested(
3935        &self,
3936        ready_timeout: Option<Duration>,
3937        requester: Option<Principal>,
3938    ) -> Result<(), SuperviseError> {
3939        let (reply_tx, reply_rx) = oneshot::channel();
3940        self.inner
3941            .commands
3942            .send(SupervisorCommand::Swap {
3943                requester,
3944                ready_timeout,
3945                reply: reply_tx,
3946            })
3947            .await
3948            .map_err(|_| SuperviseError::CommandClosed {
3949                module_id: self.inner.module_id.clone(),
3950            })?;
3951        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3952            module_id: self.inner.module_id.clone(),
3953        })?
3954    }
3955
3956    pub async fn reload(&self) -> Result<(), SuperviseError> {
3957        self.reload_requested(None).await
3958    }
3959
3960    pub(crate) async fn reload_requested(
3961        &self,
3962        requester: Option<Principal>,
3963    ) -> Result<(), SuperviseError> {
3964        let (reply_tx, reply_rx) = oneshot::channel();
3965        self.inner
3966            .commands
3967            .send(SupervisorCommand::Reload {
3968                requester,
3969                reply: reply_tx,
3970            })
3971            .await
3972            .map_err(|_| SuperviseError::CommandClosed {
3973                module_id: self.inner.module_id.clone(),
3974            })?;
3975        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3976            module_id: self.inner.module_id.clone(),
3977        })?
3978    }
3979
3980    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3981        self.set_enabled_requested(enabled, None).await
3982    }
3983
3984    pub(crate) async fn set_enabled_requested(
3985        &self,
3986        enabled: bool,
3987        requester: Option<Principal>,
3988    ) -> Result<bool, SuperviseError> {
3989        let (reply_tx, reply_rx) = oneshot::channel();
3990        self.inner
3991            .commands
3992            .send(SupervisorCommand::SetEnabled {
3993                requester,
3994                enabled,
3995                reply: reply_tx,
3996            })
3997            .await
3998            .map_err(|_| SuperviseError::CommandClosed {
3999                module_id: self.inner.module_id.clone(),
4000            })?;
4001        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4002            module_id: self.inner.module_id.clone(),
4003        })?
4004    }
4005
4006    /// The current process's protocol, or the configured protocol when down.
4007    /// A rescan stores the next launch spec without changing how an existing
4008    /// process registers, serves routes, is probed, or exits.
4009    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4010        let configured = self
4011            .inner
4012            .configuration
4013            .lock()
4014            .map_err(|_| SuperviseError::StatePoisoned {
4015                module_id: Some(self.inner.module_id.clone()),
4016            })?
4017            .spec
4018            .protocol;
4019        let state = lock_snapshot(&self.inner.snapshot)?;
4020        Ok(state.spawned_protocol.unwrap_or(configured))
4021    }
4022
4023    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4024        let configuration =
4025            self.inner
4026                .configuration
4027                .lock()
4028                .map_err(|_| SuperviseError::StatePoisoned {
4029                    module_id: Some(self.inner.module_id.clone()),
4030                })?;
4031        Ok((configuration.spec.clone(), configuration.health.clone()))
4032    }
4033
4034    /// Replace this module's launch spec, keeping its health and drain policy,
4035    /// the way a rescan does for a changed config entry. The running process is
4036    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4037    #[cfg(any(test, feature = "test-support"))]
4038    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4039        let (_, health) = self.configuration()?;
4040        let drain_timeout_ms = u64::try_from(
4041            self.inner
4042                .effective_drain_timeout
4043                .lock()
4044                .unwrap_or_else(|poisoned| poisoned.into_inner())
4045                .as_millis(),
4046        )
4047        .ok();
4048        self.update_configuration(spec, health, drain_timeout_ms)
4049            .await
4050    }
4051
4052    pub(crate) async fn update_configuration(
4053        &self,
4054        spec: ModuleSpec,
4055        health: HealthConfig,
4056        drain_timeout_ms: Option<u64>,
4057    ) -> Result<(), SuperviseError> {
4058        if spec.module_id != self.inner.module_id {
4059            return Err(SuperviseError::InvalidSpec {
4060                reason: "a supervised module's module_id cannot be changed".to_string(),
4061            });
4062        }
4063        validate_spec(&spec)?;
4064        let (reply_tx, reply_rx) = oneshot::channel();
4065        self.inner
4066            .commands
4067            .send(SupervisorCommand::UpdateConfiguration {
4068                spec: spec.clone(),
4069                health: health.clone(),
4070                drain_timeout_ms,
4071                reply: reply_tx,
4072            })
4073            .await
4074            .map_err(|_| SuperviseError::CommandClosed {
4075                module_id: self.inner.module_id.clone(),
4076            })?;
4077        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4078            module_id: self.inner.module_id.clone(),
4079        })?;
4080        let mut configuration =
4081            self.inner
4082                .configuration
4083                .lock()
4084                .map_err(|_| SuperviseError::StatePoisoned {
4085                    module_id: Some(self.inner.module_id.clone()),
4086                })?;
4087        configuration.spec = spec;
4088        configuration.health = health;
4089        Ok(())
4090    }
4091}
4092
4093impl Drop for SupervisedModuleInner {
4094    fn drop(&mut self) {
4095        let Ok(mut monitor) = self.monitor.lock() else {
4096            return;
4097        };
4098        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4099            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4100                state.state = ModuleState::Stopped;
4101                clear_current_process_facts(state);
4102            });
4103            monitor.abort();
4104        }
4105        let _ = monitor.take();
4106    }
4107}
4108
4109#[derive(Debug)]
4110enum SupervisorCommand {
4111    Drain {
4112        requester: Option<Principal>,
4113        reply: oneshot::Sender<Result<(), SuperviseError>>,
4114    },
4115    Retire {
4116        requester: Option<Principal>,
4117        reply: oneshot::Sender<Result<(), SuperviseError>>,
4118    },
4119    Restart {
4120        requester: Option<Principal>,
4121        /// Operator override for this one restart's drain budget, in ms. `None`
4122        /// uses the module's configured/default budget; `Some(0)` cuts
4123        /// immediately (wedge bounce: a stuck request never settles, so
4124        /// waiting only delays recovery).
4125        drain_timeout_ms: Option<u64>,
4126        /// The module's `spawn_generation` when the request was received, before
4127        /// it waited in the command queue. A queued restart whose module has
4128        /// since spawned a newer process is already satisfied (see the handler).
4129        received_at_generation: u64,
4130        /// When the request entered the command queue, so the handler can log
4131        /// how long it waited behind the loop's other work.
4132        queued_at: Instant,
4133        reply: oneshot::Sender<Result<(), SuperviseError>>,
4134    },
4135    Reload {
4136        requester: Option<Principal>,
4137        reply: oneshot::Sender<Result<(), SuperviseError>>,
4138    },
4139    SetEnabled {
4140        requester: Option<Principal>,
4141        enabled: bool,
4142        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4143    },
4144    UpdateConfiguration {
4145        spec: ModuleSpec,
4146        health: HealthConfig,
4147        /// Per-module drain override from the new config; `None` re-resolves to
4148        /// the supervisor-wide default.
4149        drain_timeout_ms: Option<u64>,
4150        reply: oneshot::Sender<()>,
4151    },
4152    Swap {
4153        requester: Option<Principal>,
4154        /// How long the candidate may take to register and declare itself
4155        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4156        ready_timeout: Option<Duration>,
4157        /// Answered at cutover or failure; the incumbent's drain follows.
4158        reply: oneshot::Sender<Result<(), SuperviseError>>,
4159    },
4160}
4161
4162#[derive(Debug)]
4163pub enum SuperviseError {
4164    InvalidSpec {
4165        reason: String,
4166    },
4167    Spawn {
4168        program: PathBuf,
4169        source: io::Error,
4170        cgroup_path: Option<PathBuf>,
4171    },
4172    Cgroup {
4173        module_id: String,
4174        source: io::Error,
4175    },
4176    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4177    /// than spawn a reserved module without its identity binding.
4178    LaunchNonce {
4179        reason: String,
4180    },
4181    Wait {
4182        module_id: String,
4183        source: io::Error,
4184    },
4185    Kill {
4186        module_id: String,
4187        source: io::Error,
4188    },
4189    Forwarding(ForwardingError),
4190    Registry(RegistryError),
4191    ReloadUnavailable {
4192        module_id: String,
4193        reason: String,
4194    },
4195    /// An operator restart/reload was requested for a module that is currently
4196    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4197    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4198    /// by a restart, so these commands are rejected instead of re-enabling it.
4199    Disabled {
4200        module_id: String,
4201    },
4202    ReloadFailed {
4203        module_id: String,
4204        reason: String,
4205    },
4206    RegistrationStillActive {
4207        module_id: String,
4208        waited: Duration,
4209    },
4210    StatePoisoned {
4211        module_id: Option<String>,
4212    },
4213    CommandClosed {
4214        module_id: String,
4215    },
4216    /// A restart or reload arrived while a swap's candidate was warming. The
4217    /// swap owns the module until it cuts over or fails; a stop or disable
4218    /// would have aborted it instead.
4219    SwapInProgress {
4220        module_id: String,
4221    },
4222    /// A swap was refused before anything was spawned.
4223    SwapRefused {
4224        module_id: String,
4225        reason: SwapRefusal,
4226    },
4227    /// A swap spawned a candidate and gave up on it. The candidate has been
4228    /// killed and its slot freed; the incumbent was left serving and was never
4229    /// drained, except in the one `CutoverLost` case described on that arm.
4230    SwapFailed {
4231        module_id: String,
4232        arm: SwapFailureArm,
4233        detail: String,
4234        /// How the candidate exited, when it exited on its own before the
4235        /// supervisor gave up on it.
4236        candidate_exit: Option<ExitReport>,
4237    },
4238}
4239
4240/// Why a swap was refused before a candidate was spawned.
4241#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4242pub enum SwapRefusal {
4243    /// The module's config does not declare `overlap: "safe"`.
4244    OverlapExclusive,
4245    /// The module is not registered, so there is no incumbent to keep serving
4246    /// and nothing a swap would improve on; a plain restart is the tool.
4247    NotRegistered,
4248    /// The module does not speak the subc wire, so a candidate could never
4249    /// register or declare itself ready.
4250    ProtocolNone,
4251    /// The supervisor lacks the forwarding table (to cut routes over) or the
4252    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4253    NotConfigured,
4254    /// A swap is already open for this module.
4255    AlreadySwapping,
4256}
4257
4258impl SwapRefusal {
4259    pub fn as_str(self) -> &'static str {
4260        match self {
4261            Self::OverlapExclusive => "overlap_exclusive",
4262            Self::NotRegistered => "not_registered",
4263            Self::ProtocolNone => "protocol_none",
4264            Self::NotConfigured => "not_configured",
4265            Self::AlreadySwapping => "already_swapping",
4266        }
4267    }
4268}
4269
4270/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4271/// serving and undrained; see `CutoverLost`.
4272#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4273pub enum SwapFailureArm {
4274    /// The candidate process could not be started.
4275    SpawnFailed,
4276    /// The candidate did not register within the readiness budget.
4277    NeverRegistered,
4278    /// The candidate registered but did not declare itself ready in time.
4279    NeverReady,
4280    /// The candidate exited before cutover.
4281    CandidateExited,
4282    /// The candidate declared itself ready but failed its health probe.
4283    CandidateUnhealthy,
4284    /// An operator stop, disable or retire arrived while the candidate warmed.
4285    /// The candidate was killed and the operator's command then carried out on
4286    /// the incumbent.
4287    Interrupted,
4288    /// The candidate's connection closed at the moment of cutover. If it
4289    /// closed before forwarding moved, the incumbent is untouched. If it closed
4290    /// between the forwarding and registry halves of cutover, forwarding can no
4291    /// longer route to the incumbent, so the module is restarted plainly.
4292    CutoverLost,
4293}
4294
4295impl SwapFailureArm {
4296    pub fn as_str(self) -> &'static str {
4297        match self {
4298            Self::SpawnFailed => "spawn_failed",
4299            Self::NeverRegistered => "never_registered",
4300            Self::NeverReady => "never_ready",
4301            Self::CandidateExited => "candidate_exited",
4302            Self::CandidateUnhealthy => "candidate_unhealthy",
4303            Self::Interrupted => "interrupted",
4304            Self::CutoverLost => "cutover_lost",
4305        }
4306    }
4307}
4308
4309impl fmt::Display for SuperviseError {
4310    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4311        match self {
4312            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4313            Self::Spawn {
4314                program,
4315                source,
4316                cgroup_path: Some(cgroup_path),
4317            } => write!(
4318                f,
4319                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4320                cgroup_path.display(),
4321                program.display()
4322            ),
4323            Self::Spawn {
4324                program,
4325                source,
4326                cgroup_path: None,
4327            } => write!(
4328                f,
4329                "failed to spawn module '{}': {source}",
4330                program.display()
4331            ),
4332            Self::Cgroup { module_id, source } => {
4333                write!(
4334                    f,
4335                    "failed to prepare cgroup for module '{module_id}': {source}"
4336                )
4337            }
4338            Self::LaunchNonce { reason } => {
4339                write!(
4340                    f,
4341                    "failed to generate reserved-module launch nonce: {reason}"
4342                )
4343            }
4344            Self::Wait { module_id, source } => {
4345                write!(f, "failed to wait for module '{module_id}': {source}")
4346            }
4347            Self::Kill { module_id, source } => {
4348                write!(f, "failed to kill module '{module_id}': {source}")
4349            }
4350            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4351            Self::Registry(err) => write!(f, "registry error: {err}"),
4352            Self::ReloadUnavailable { module_id, reason } => {
4353                write!(f, "reload unavailable for module '{module_id}': {reason}")
4354            }
4355            Self::Disabled { module_id } => {
4356                write!(
4357                    f,
4358                    "module '{module_id}' is disabled; enable it before restart or reload"
4359                )
4360            }
4361            Self::ReloadFailed { module_id, reason } => {
4362                write!(f, "reload failed for module '{module_id}': {reason}")
4363            }
4364            Self::RegistrationStillActive { module_id, waited } => write!(
4365                f,
4366                "module '{module_id}' registration remained active after waiting {waited:?}"
4367            ),
4368            Self::StatePoisoned { module_id } => match module_id {
4369                Some(module_id) => {
4370                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4371                }
4372                None => write!(f, "supervisor state was poisoned"),
4373            },
4374            Self::CommandClosed { module_id } => {
4375                write!(
4376                    f,
4377                    "supervisor command channel for module '{module_id}' is closed"
4378                )
4379            }
4380            Self::SwapInProgress { module_id } => write!(
4381                f,
4382                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4383            ),
4384            Self::SwapRefused { module_id, reason } => match reason {
4385                SwapRefusal::OverlapExclusive => write!(
4386                    f,
4387                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4388                ),
4389                SwapRefusal::NotRegistered => write!(
4390                    f,
4391                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4392                ),
4393                SwapRefusal::ProtocolNone => write!(
4394                    f,
4395                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4396                ),
4397                SwapRefusal::NotConfigured => write!(
4398                    f,
4399                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4400                ),
4401                SwapRefusal::AlreadySwapping => {
4402                    write!(f, "module '{module_id}' is already being swapped")
4403                }
4404            },
4405            Self::SwapFailed {
4406                module_id,
4407                arm,
4408                detail,
4409                ..
4410            } => write!(
4411                f,
4412                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4413                arm.as_str()
4414            ),
4415        }
4416    }
4417}
4418
4419impl Error for SuperviseError {
4420    fn source(&self) -> Option<&(dyn Error + 'static)> {
4421        match self {
4422            Self::Spawn { source, .. }
4423            | Self::Cgroup { source, .. }
4424            | Self::Wait { source, .. }
4425            | Self::Kill { source, .. } => Some(source),
4426            Self::Forwarding(err) => Some(err),
4427            Self::Registry(err) => Some(err),
4428            Self::LaunchNonce { .. }
4429            | Self::InvalidSpec { .. }
4430            | Self::ReloadUnavailable { .. }
4431            | Self::Disabled { .. }
4432            | Self::ReloadFailed { .. }
4433            | Self::RegistrationStillActive { .. }
4434            | Self::StatePoisoned { .. }
4435            | Self::CommandClosed { .. }
4436            | Self::SwapInProgress { .. }
4437            | Self::SwapRefused { .. }
4438            | Self::SwapFailed { .. } => None,
4439        }
4440    }
4441}
4442
4443pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4444    if spec.module_id.trim().is_empty() {
4445        return Err(SuperviseError::InvalidSpec {
4446            reason: "module_id must not be empty".to_string(),
4447        });
4448    }
4449
4450    Ok(())
4451}
4452
4453#[derive(Debug, Default)]
4454struct HealthProbeRuntime {
4455    configured_health: Option<HealthConfig>,
4456    registered_connection: Option<crate::ConnectionId>,
4457    advertised: bool,
4458    next_probe_at: Option<Instant>,
4459    probe_index: u64,
4460}
4461
4462fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4463    lock_snapshot(snapshot)
4464        .ok()
4465        .and_then(|state| state.spawned_protocol)
4466        .unwrap_or(spec.protocol)
4467}
4468
4469impl HealthProbeRuntime {
4470    fn refresh_registration(
4471        &mut self,
4472        spec: &ModuleSpec,
4473        runtime: &SupervisorRuntimeConfig,
4474        registry: &Registry,
4475        snapshot: &SharedSnapshot,
4476    ) {
4477        if self.configured_health.as_ref() != Some(&runtime.health) {
4478            self.configured_health = Some(runtime.health.clone());
4479            self.next_probe_at = None;
4480            self.registered_connection = None;
4481            self.probe_index = 0;
4482        }
4483        // A non-wire process never registers. Only an explicitly configured
4484        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4485        // health failure for that kind of process.
4486        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4487            self.registered_connection = None;
4488            self.advertised = runtime.health.http.is_some();
4489            if !self.advertised {
4490                self.next_probe_at = None;
4491                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4492                    let unknown = ModuleHealthStatus::default();
4493                    if state.health != unknown {
4494                        state.health = unknown;
4495                    }
4496                });
4497            } else if self.next_probe_at.is_none() {
4498                self.next_probe_at = Some(
4499                    Instant::now()
4500                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4501                );
4502            }
4503            return;
4504        }
4505
4506        let registration = match registry.get_module(&spec.module_id) {
4507            Ok(registration) => registration,
4508            Err(err) => {
4509                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4510                self.advertised = false;
4511                self.next_probe_at = None;
4512                return;
4513            }
4514        };
4515
4516        let Some(registration) = registration else {
4517            self.registered_connection = None;
4518            self.advertised = false;
4519            self.next_probe_at = None;
4520            return;
4521        };
4522
4523        let advertised = registration
4524            .control_ops
4525            .iter()
4526            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4527        if !advertised {
4528            self.registered_connection = Some(registration.connection_id);
4529            self.advertised = false;
4530            self.next_probe_at = None;
4531            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4532                if state.health.status != SupervisorHealthStatus::Unknown
4533                    || state.health.consecutive_failures != 0
4534                    || state.health.last_probe_ms.is_some()
4535                    || state.health.detail.is_some()
4536                    || state.health.metrics.is_some()
4537                {
4538                    state.health.status = SupervisorHealthStatus::Unknown;
4539                    state.health.consecutive_failures = 0;
4540                    state.health.last_probe_ms = None;
4541                    state.health.detail = None;
4542                    state.health.metrics = None;
4543                }
4544            });
4545            return;
4546        }
4547
4548        let reregistered = self.registered_connection != Some(registration.connection_id);
4549        self.registered_connection = Some(registration.connection_id);
4550        self.advertised = true;
4551        if reregistered || self.next_probe_at.is_none() {
4552            self.probe_index = 0;
4553            self.next_probe_at = Some(
4554                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4555            );
4556            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4557                state.health.status = SupervisorHealthStatus::Unknown;
4558                state.health.consecutive_failures = 0;
4559                state.health.detail = None;
4560                state.health.metrics = None;
4561            });
4562        }
4563    }
4564
4565    fn wake_after(&self) -> Option<Duration> {
4566        if !self.advertised {
4567            return None;
4568        }
4569        self.next_probe_at
4570            .map(|next| next.saturating_duration_since(Instant::now()))
4571    }
4572
4573    fn due(&self) -> bool {
4574        self.advertised
4575            && self
4576                .next_probe_at
4577                .is_some_and(|next| Instant::now() >= next)
4578    }
4579
4580    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4581        self.probe_index = self.probe_index.wrapping_add(1);
4582        self.next_probe_at = Some(
4583            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4584        );
4585    }
4586}
4587
4588/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4589///
4590/// This was a struct with a single `message: String`, and every one of the
4591/// fifteen construction sites collapsed into it. Each site knows exactly what it
4592/// saw -- the lane is gone, the module did not answer in time, the module
4593/// answered with the wrong thing -- and `handle_health_probe_failure` then
4594/// treated all of them identically: increment a counter, compare to a threshold,
4595/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4596/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4597///
4598/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4599///
4600/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4601///   answer on it again.
4602/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4603///   AND with a perfectly healthy one that lost a CPU race -- which is what
4604///   happens under machine load, and is how this supervisor killed a healthy
4605///   module three times in one day.
4606/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4607///   Restarting on it is defensible, but it is not the silence case and should
4608///   never be counted as one.
4609/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4610///   anything, so it cannot be evidence about the module at all.
4611///
4612/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4613/// one that fires most often, and while every variant collapsed into one string
4614/// it carried the same weight as the strongest.
4615///
4616/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4617/// DESIGN and a reader stopping at it gets the build backwards: the restart
4618/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4619/// probes still increment the failure streak and drive escalation at the
4620/// threshold (see `is_proof_of_death` below for why that is deliberate and
4621/// what gates the change). Absence of evidence restarts modules today.
4622#[derive(Debug)]
4623enum HealthProbeEvidence {
4624    /// The module's control lane is gone. Proof of death.
4625    LaneDead,
4626    /// No reply within the deadline. Proves nothing about the module's state.
4627    NoAnswer,
4628    /// The module replied, but not with a usable health report. Proves it is alive.
4629    BadAnswer,
4630    /// The daemon could not ask. Says nothing about the module.
4631    Misconfigured,
4632}
4633
4634#[derive(Debug)]
4635struct HealthProbeError {
4636    evidence: HealthProbeEvidence,
4637    message: String,
4638}
4639
4640impl HealthProbeError {
4641    fn lane_dead(message: impl Into<String>) -> Self {
4642        Self::with(HealthProbeEvidence::LaneDead, message)
4643    }
4644
4645    fn no_answer(message: impl Into<String>) -> Self {
4646        Self::with(HealthProbeEvidence::NoAnswer, message)
4647    }
4648
4649    fn bad_answer(message: impl Into<String>) -> Self {
4650        Self::with(HealthProbeEvidence::BadAnswer, message)
4651    }
4652
4653    fn misconfigured(message: impl Into<String>) -> Self {
4654        Self::with(HealthProbeEvidence::Misconfigured, message)
4655    }
4656
4657    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4658        Self {
4659            evidence,
4660            message: message.into(),
4661        }
4662    }
4663
4664    /// Whether this observation is proof the module cannot serve.
4665    ///
4666    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4667    /// variant that fires under CPU starvation, and treating it as proof is the
4668    /// defect this enum exists to make impossible to reintroduce silently.
4669    ///
4670    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4671    /// to restart also needs a bound for the case it excludes -- a genuinely
4672    /// wedged module, alive but never answering -- and that bound must come from
4673    /// the distribution of real late-answer latencies, which nothing measures
4674    /// yet. Landing the classification first makes the later change a one-line
4675    /// decision against evidence that already exists, rather than two unproven
4676    /// changes at once.
4677    #[allow(dead_code)]
4678    fn is_proof_of_death(&self) -> bool {
4679        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4680    }
4681
4682    /// Short stable label for logs and the health snapshot.
4683    ///
4684    /// An operator reading `ck health` currently cannot tell "the module is gone"
4685    /// from "the module did not answer in five seconds", because both render as
4686    /// prose in the same field. These labels are what make the two
4687    /// distinguishable at a glance, and they are what a later restart-policy
4688    /// change will be argued from.
4689    fn label(&self) -> &'static str {
4690        match self.evidence {
4691            HealthProbeEvidence::LaneDead => "lane-dead",
4692            HealthProbeEvidence::NoAnswer => "no-answer",
4693            HealthProbeEvidence::BadAnswer => "bad-answer",
4694            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4695        }
4696    }
4697}
4698
4699impl fmt::Display for HealthProbeError {
4700    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4701        f.write_str(&self.message)
4702    }
4703}
4704
4705async fn run_health_probe_cycle(
4706    spec: &ModuleSpec,
4707    runtime: &SupervisorRuntimeConfig,
4708    registry: &Registry,
4709    process_liveness: &SupervisorProcessLiveness,
4710    snapshot: &SharedSnapshot,
4711    child: &mut Option<SupervisedChild>,
4712) {
4713    let now_ms = unix_ms_now();
4714    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4715        .then_some(runtime.health.http.as_deref())
4716        .flatten();
4717    let result = match http {
4718        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4719        None => probe_module_health(&spec.module_id, runtime, None).await,
4720    };
4721    match result {
4722        Ok(report) => {
4723            handle_health_report(
4724                spec,
4725                runtime,
4726                registry,
4727                process_liveness,
4728                snapshot,
4729                child,
4730                report,
4731                now_ms,
4732            )
4733            .await;
4734        }
4735        Err(err) => {
4736            if http.is_some() {
4737                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4738                    state.health.status = SupervisorHealthStatus::Failing;
4739                });
4740            }
4741            handle_health_probe_failure(
4742                spec,
4743                runtime,
4744                registry,
4745                process_liveness,
4746                snapshot,
4747                child,
4748                err,
4749                now_ms,
4750            )
4751            .await;
4752        }
4753    }
4754}
4755
4756pub(crate) struct HttpProbeTarget<'a> {
4757    address: std::net::SocketAddr,
4758    localhost: bool,
4759    authority: &'a str,
4760    path: String,
4761}
4762
4763/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4764/// or TLS. A URL cannot turn a local health check into an outbound connection.
4765pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4766    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4767        return Err("must not contain whitespace, controls, or a fragment".into());
4768    }
4769    let rest = url
4770        .strip_prefix("http://")
4771        .ok_or("must use plain http://")?;
4772    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4773    let (authority, suffix) = rest.split_at(split);
4774    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4775        ("::1", rest)
4776    } else {
4777        let split = authority.find(':').unwrap_or(authority.len());
4778        authority.split_at(split)
4779    };
4780    let ip: std::net::IpAddr = match host {
4781        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4782        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4783        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4784    };
4785    let port = if port.is_empty() {
4786        80
4787    } else {
4788        port.strip_prefix(':')
4789            .and_then(|p| p.parse::<u16>().ok())
4790            .filter(|p| *p > 0)
4791            .ok_or("must have a valid nonzero TCP port")?
4792    };
4793    let path = if suffix.is_empty() {
4794        "/".into()
4795    } else if suffix.starts_with('?') {
4796        format!("/{suffix}")
4797    } else {
4798        suffix.into()
4799    };
4800    Ok(HttpProbeTarget {
4801        address: std::net::SocketAddr::new(ip, port),
4802        localhost: host == "localhost",
4803        authority,
4804        path,
4805    })
4806}
4807
4808async fn probe_http_health(
4809    url: &str,
4810    deadline: Duration,
4811) -> Result<HealthReport, HealthProbeError> {
4812    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4813    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4814    // Keep partial diagnostics outside the timed future so cancellation does
4815    // not discard a status line or body bytes already received.
4816    let mut response_status = String::new();
4817    let mut body = Vec::new();
4818    let probe = async {
4819        // Resolve localhost ourselves so a hosts-file override cannot turn
4820        // this into an outbound request, while IPv6-only local servers work.
4821        let connection = match tokio::net::TcpStream::connect(target.address).await {
4822            Err(_) if target.localhost => {
4823                tokio::net::TcpStream::connect((
4824                    std::net::Ipv6Addr::LOCALHOST,
4825                    target.address.port(),
4826                ))
4827                .await
4828            }
4829            result => result,
4830        };
4831        let mut stream = connection.map_err(|error| {
4832            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4833        })?;
4834        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4835            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4836        let mut reader = BufReader::new(stream);
4837        let mut budget = 16 * 1024;
4838        let status = http_line(&mut reader, &mut budget).await?;
4839        let mut words = status.split_ascii_whitespace();
4840        let version = words.next();
4841        let code = words
4842            .next()
4843            .filter(|word| word.len() == 3)
4844            .and_then(|word| word.parse::<u16>().ok());
4845        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4846            || !code.is_some_and(|code| (100..600).contains(&code))
4847        {
4848            return Err(HealthProbeError::bad_answer(format!(
4849                "invalid HTTP status: {status}"
4850            )));
4851        }
4852        let code = code.expect("validated status code");
4853        response_status = status.clone();
4854        let mut length = None;
4855        let mut chunked = false;
4856        loop {
4857            let line = http_line(&mut reader, &mut budget).await?;
4858            if line.is_empty() {
4859                break;
4860            }
4861            if let Some((name, value)) = line.split_once(':') {
4862                if name.eq_ignore_ascii_case("content-length") {
4863                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4864                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4865                    })?);
4866                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4867                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4868                }
4869            }
4870        }
4871        if chunked {
4872            while body.len() < 200 {
4873                let line = http_line(&mut reader, &mut budget).await?;
4874                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4875                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4876                if size == 0 {
4877                    break;
4878                }
4879                let count = size.min((200 - body.len()) as u64) as usize;
4880                let start = body.len();
4881                (&mut reader)
4882                    .take(count as u64)
4883                    .read_to_end(&mut body)
4884                    .await
4885                    .map_err(|error| {
4886                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4887                    })?;
4888                if body.len() - start != count {
4889                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4890                }
4891                if size > count as u64 || body.len() == 200 {
4892                    break;
4893                }
4894                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4895                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4896                }
4897            }
4898        } else {
4899            reader
4900                .take(length.unwrap_or(200).min(200))
4901                .read_to_end(&mut body)
4902                .await
4903                .map_err(|error| {
4904                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4905                })?;
4906        }
4907        if (200..300).contains(&code) {
4908            Ok(HealthReport::ok())
4909        } else {
4910            Err(HealthProbeError::bad_answer(
4911                "HTTP health endpoint returned non-2xx",
4912            ))
4913        }
4914    };
4915    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4916        Err(HealthProbeError::no_answer(format!(
4917            "HTTP probe timed out after {deadline:?}"
4918        )))
4919    });
4920    if let Err(error) = &mut result {
4921        if !response_status.is_empty() {
4922            error.message = format!(
4923                "{}; {response_status}: {}",
4924                error.message,
4925                String::from_utf8_lossy(&body)
4926            );
4927        }
4928    }
4929    result
4930}
4931
4932async fn http_line(
4933    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4934    remaining: &mut usize,
4935) -> Result<String, HealthProbeError> {
4936    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4937    let mut line = Vec::new();
4938    (&mut *reader)
4939        .take(*remaining as u64)
4940        .read_until(b'\n', &mut line)
4941        .await
4942        .map_err(|error| {
4943            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4944        })?;
4945    *remaining -= line.len();
4946    if !line.ends_with(b"\r\n") {
4947        return Err(HealthProbeError::bad_answer(
4948            "HTTP headers are incomplete or exceed 16 KiB",
4949        ));
4950    }
4951    line.truncate(line.len() - 2);
4952    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4953}
4954
4955async fn probe_module_health(
4956    module_id: &str,
4957    runtime: &SupervisorRuntimeConfig,
4958    drain_deadline: Option<Instant>,
4959) -> Result<HealthReport, HealthProbeError> {
4960    let Some(forwarding) = runtime.forwarding.as_ref() else {
4961        return Err(HealthProbeError::misconfigured(
4962            "supervisor was not configured with a forwarding table",
4963        ));
4964    };
4965    let probe_started_at = Instant::now();
4966    let mut deadline = probe_started_at + runtime.health.deadline;
4967    if let Some(drain_deadline) = drain_deadline {
4968        deadline = deadline.min(drain_deadline);
4969    }
4970    let pending = if drain_deadline.is_some() {
4971        forwarding.begin_drain_health_probe_rpc_for(
4972            module_id,
4973            MODULE_CONTROL_OP_HEALTH_CHECK,
4974            probe_started_at,
4975            deadline,
4976        )
4977    } else {
4978        forwarding.begin_health_probe_rpc_for(
4979            module_id,
4980            MODULE_CONTROL_OP_HEALTH_CHECK,
4981            probe_started_at,
4982            deadline,
4983        )
4984    }
4985    .map_err(|err| {
4986        // The endpoint is not registered, so there is no live control lane to
4987        // ask. That is the module being absent, not slow.
4988        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4989    })?;
4990    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4991}
4992
4993/// [`probe_module_health`] for one endpoint rather than the id's active one.
4994///
4995/// A swap probes two processes that no by-id lookup reaches: its candidate
4996/// before cutover, and its superseded incumbent (for busy gauges) while the
4997/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4998/// bounds the by-id drain probe.
4999async fn probe_endpoint_health(
5000    endpoint: crate::ModuleEndpointId,
5001    runtime: &SupervisorRuntimeConfig,
5002    deadline_cap: Option<Instant>,
5003) -> Result<HealthReport, HealthProbeError> {
5004    let Some(forwarding) = runtime.forwarding.as_ref() else {
5005        return Err(HealthProbeError::misconfigured(
5006            "supervisor was not configured with a forwarding table",
5007        ));
5008    };
5009    let probe_started_at = Instant::now();
5010    let mut deadline = probe_started_at + runtime.health.deadline;
5011    if let Some(cap) = deadline_cap {
5012        deadline = deadline.min(cap);
5013    }
5014    let pending = forwarding
5015        .begin_endpoint_health_probe_rpc_for(
5016            endpoint,
5017            MODULE_CONTROL_OP_HEALTH_CHECK,
5018            probe_started_at,
5019            deadline,
5020        )
5021        .map_err(|err| {
5022            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5023        })?;
5024    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5025}
5026
5027/// Send a begun health probe and classify its answer.
5028async fn await_health_probe(
5029    forwarding: &ForwardingTable,
5030    pending: PendingModuleControlRpc,
5031    deadline: Instant,
5032    probe_budget: Duration,
5033) -> Result<HealthReport, HealthProbeError> {
5034    let PendingModuleControlRpc {
5035        endpoint,
5036        module_sink,
5037        negotiated_ver,
5038        corr,
5039        receiver,
5040    } = pending;
5041    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5042        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5043    })?;
5044    let frame = Frame::build_with_version(
5045        negotiated_ver,
5046        FrameType::Request,
5047        control_flags(),
5048        0,
5049        0,
5050        corr,
5051        body,
5052    )
5053    .map_err(|err| {
5054        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5055    })?;
5056
5057    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5058    // blocks waiting for capacity when the module's egress queue is full, and an
5059    // unbounded await here freezes the whole supervision actor (it stops polling
5060    // Child::wait and supervisor commands), making the module unrecoverable
5061    // in-band. On timeout the probe fails like any transport failure.
5062    match timeout_at(deadline, module_sink.send(frame)).await {
5063        Ok(Ok(())) => {}
5064        Ok(Err(err)) => {
5065            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5066            // A closed sink means the module's egress channel is gone -- the
5067            // receiving half is dropped when its connection tears down. Proof.
5068            return Err(HealthProbeError::lane_dead(format!(
5069                "failed to send health.check: {err}"
5070            )));
5071        }
5072        Err(_elapsed) => {
5073            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5074            // A full egress queue means the module is not draining its socket, which
5075            // is consistent with a wedged module AND with one whose reader is merely
5076            // starved. Silence, not proof.
5077            return Err(HealthProbeError::no_answer(
5078                "health.check send timed out before enqueue (module egress full)",
5079            ));
5080        }
5081    }
5082
5083    match timeout_at(deadline, receiver).await {
5084        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5085        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5086        // and those prove it is alive even though the probe failed.
5087        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5088            response.health_report().ok_or_else(|| {
5089                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5090            })
5091        }
5092        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5093            format!("health.check rejected: {}", body.message),
5094        )),
5095        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5096            Err(HealthProbeError::lane_dead(message))
5097        }
5098        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5099            Err(HealthProbeError::bad_answer(message))
5100        }
5101        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5102            Err(HealthProbeError::bad_answer(format!(
5103                "expected module-control op '{expected}', got '{actual}'"
5104            )))
5105        }
5106        // A reply that crosses the deadline before this waiter observes it is
5107        // still proof of life. The forwarding path records its end-to-end latency
5108        // before delivering this classification.
5109        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5110            "module answered health.check after its daemon deadline",
5111        )),
5112        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5113            "health.check waiter was canceled before the module responded",
5114        )),
5115        Err(_) => {
5116            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5117            Err(HealthProbeError::no_answer(format!(
5118                "module did not answer health.check within {probe_budget:?}"
5119            )))
5120        }
5121    }
5122}
5123
5124#[allow(clippy::too_many_arguments)]
5125async fn handle_health_report(
5126    spec: &ModuleSpec,
5127    runtime: &SupervisorRuntimeConfig,
5128    registry: &Registry,
5129    process_liveness: &SupervisorProcessLiveness,
5130    snapshot: &SharedSnapshot,
5131    child: &mut Option<SupervisedChild>,
5132    report: HealthReport,
5133    now_ms: u64,
5134) {
5135    let status = supervisor_health_status(report.status);
5136    let detail = report.detail.clone();
5137    let metrics = truncate_health_metrics(report.metrics);
5138    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5139        state.health.status = status;
5140        state.health.last_probe_ms = Some(now_ms);
5141        state.health.detail = detail.clone();
5142        state.health.metrics = metrics.clone();
5143        state.health.consecutive_failures = 0;
5144    });
5145
5146    let action = match report.status {
5147        HealthStatus::Ok => return,
5148        HealthStatus::Degraded => runtime.health.on_degraded,
5149        HealthStatus::Failing => runtime.health.on_failing,
5150    };
5151    apply_l3_health_action(
5152        spec,
5153        runtime,
5154        registry,
5155        process_liveness,
5156        snapshot,
5157        child,
5158        status,
5159        detail.as_deref(),
5160        action,
5161        now_ms,
5162    )
5163    .await;
5164}
5165
5166#[allow(clippy::too_many_arguments)]
5167async fn handle_health_probe_failure(
5168    spec: &ModuleSpec,
5169    runtime: &SupervisorRuntimeConfig,
5170    registry: &Registry,
5171    process_liveness: &SupervisorProcessLiveness,
5172    snapshot: &SharedSnapshot,
5173    child: &mut Option<SupervisedChild>,
5174    err: HealthProbeError,
5175    now_ms: u64,
5176) {
5177    let threshold = runtime.health.failure_threshold.max(1);
5178    let mut failures = 0;
5179    // Carry the evidence class into the operator-visible detail. Without it,
5180    // "module did not answer within 5s" and "the control lane is gone" are two
5181    // prose strings in the same field, and the reader has to know the codebase to
5182    // tell which one is proof of anything.
5183    let detail = format!("[{}] {err}", err.label());
5184    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5185        // A failed wire probe invalidates the last report, even before the
5186        // restart threshold. HTTP probes already mark failures as Failing.
5187        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5188            state.health.status = SupervisorHealthStatus::Unknown;
5189        }
5190        state.health.last_probe_ms = Some(now_ms);
5191        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5192        state.health.detail = Some(detail.clone());
5193        state.health.metrics = None;
5194        failures = state.health.consecutive_failures;
5195    });
5196
5197    if failures < threshold {
5198        warn!(
5199            module_id = %spec.module_id,
5200            consecutive_failures = failures,
5201            threshold,
5202            evidence = err.label(),
5203            detail = %detail,
5204            "health.check probe failed"
5205        );
5206        return;
5207    }
5208
5209    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5210        state.state = ModuleState::Unresponsive;
5211        state.health.status = SupervisorHealthStatus::Unresponsive;
5212    });
5213    // The evidence class is logged at the kill site because this is the line an
5214    // operator reads after an unexplained restart. A streak of `no-answer` under
5215    // machine load is the known false-positive shape; a `lane-dead` is not.
5216    if runtime.health.critical {
5217        error!(
5218            module_id = %spec.module_id,
5219            status = "unresponsive",
5220            evidence = err.label(),
5221            detail = %detail,
5222            "critical module health alert"
5223        );
5224    } else {
5225        warn!(
5226            module_id = %spec.module_id,
5227            status = "unresponsive",
5228            evidence = err.label(),
5229            detail = %detail,
5230            "module health threshold breached"
5231        );
5232    }
5233    if let Err(err) = health_restart_child(
5234        spec,
5235        runtime,
5236        registry,
5237        process_liveness,
5238        snapshot,
5239        child,
5240        SupervisorHealthStatus::Unresponsive,
5241        Some(&detail),
5242        now_ms,
5243    )
5244    .await
5245    {
5246        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5247    }
5248}
5249
5250#[allow(clippy::too_many_arguments)]
5251async fn apply_l3_health_action(
5252    spec: &ModuleSpec,
5253    runtime: &SupervisorRuntimeConfig,
5254    registry: &Registry,
5255    process_liveness: &SupervisorProcessLiveness,
5256    snapshot: &SharedSnapshot,
5257    child: &mut Option<SupervisedChild>,
5258    status: SupervisorHealthStatus,
5259    detail: Option<&str>,
5260    action: HealthAction,
5261    now_ms: u64,
5262) {
5263    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5264    match action {
5265        HealthAction::Report => {
5266            info!(
5267                module_id = %spec.module_id,
5268                status = ?status,
5269                detail,
5270                "module reported non-ok health"
5271            );
5272        }
5273        HealthAction::Alert => {
5274            error!(
5275                module_id = %spec.module_id,
5276                status = ?status,
5277                detail,
5278                "module health alert"
5279            );
5280        }
5281        HealthAction::Restart => {
5282            if let Err(err) = health_restart_child(
5283                spec,
5284                runtime,
5285                registry,
5286                process_liveness,
5287                snapshot,
5288                child,
5289                status,
5290                detail,
5291                now_ms,
5292            )
5293            .await
5294            {
5295                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5296            }
5297        }
5298    }
5299}
5300
5301#[allow(clippy::too_many_arguments)]
5302async fn health_restart_child(
5303    spec: &ModuleSpec,
5304    runtime: &SupervisorRuntimeConfig,
5305    registry: &Registry,
5306    process_liveness: &SupervisorProcessLiveness,
5307    snapshot: &SharedSnapshot,
5308    child: &mut Option<SupervisedChild>,
5309    status: SupervisorHealthStatus,
5310    detail: Option<&str>,
5311    now_ms: u64,
5312) -> Result<(), SuperviseError> {
5313    let (enabled, schedule) = {
5314        let mut state = lock_snapshot(snapshot)?;
5315        let enabled = state.enabled;
5316        let schedule = if enabled {
5317            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5318        } else {
5319            None
5320        };
5321        (enabled, schedule)
5322    };
5323
5324    if !enabled {
5325        return Err(SuperviseError::Disabled {
5326            module_id: spec.module_id.clone(),
5327        });
5328    }
5329
5330    if schedule.is_none() {
5331        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5332        error!(
5333            module_id = %spec.module_id,
5334            status = ?status,
5335            detail,
5336            max_restarts = runtime.restart_policy.max_restarts,
5337            window_secs = runtime.restart_policy.window.as_secs(),
5338            reason = %runtime.restart_policy.budget_exhausted_detail(),
5339            "health restart budget exhausted; marking module failed"
5340        );
5341        let stop_notice = begin_forwarding_drain_if_configured(
5342            spec,
5343            runtime,
5344            registry,
5345            snapshot,
5346            Some(true),
5347            RouteCloseReason::Disable,
5348        )
5349        .await?;
5350        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5351            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5352        })?;
5353        drain_optional_child(
5354            &spec.module_id,
5355            spec.protocol,
5356            stop_notice,
5357            registry,
5358            runtime.forwarding.as_deref(),
5359            snapshot,
5360            &runtime.terminal_ring,
5361            &runtime.spawn_events,
5362            child,
5363            runtime.drain_timeout,
5364            ModuleState::Failed,
5365            Some(true),
5366        )
5367        .await?;
5368        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5369        return Ok(());
5370    }
5371
5372    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5373    let mut restart_count = 0;
5374    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5375        restart_count = state.crash_restarts.len();
5376        state.state = ModuleState::Unresponsive;
5377        state.health.status = status;
5378        state.health.last_action = Some(HealthAction::Restart.to_string());
5379        state.health.last_action_ms = Some(now_ms);
5380    })?;
5381    warn!(
5382        module_id = %spec.module_id,
5383        status = ?status,
5384        detail,
5385        restart_count,
5386        restart_in_window = schedule.restart_in_window,
5387        delay_ms = schedule.delay.as_millis() as u64,
5388        "health-triggered module restart"
5389    );
5390
5391    let stop_notice = begin_forwarding_drain_if_configured(
5392        spec,
5393        runtime,
5394        registry,
5395        snapshot,
5396        Some(true),
5397        RouteCloseReason::Restart,
5398    )
5399    .await?;
5400    drain_optional_child(
5401        &spec.module_id,
5402        spec.protocol,
5403        stop_notice,
5404        registry,
5405        runtime.forwarding.as_deref(),
5406        snapshot,
5407        &runtime.terminal_ring,
5408        &runtime.spawn_events,
5409        child,
5410        runtime.drain_timeout,
5411        ModuleState::Restarting,
5412        Some(true),
5413    )
5414    .await?;
5415    schedule_respawn(
5416        runtime,
5417        snapshot,
5418        &spec.module_id,
5419        schedule.delay,
5420        RespawnKind::Spawn,
5421    )
5422}
5423
5424fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5425    if let Some(reply) = runtime
5426        .deferred_reload_reply
5427        .lock()
5428        .unwrap_or_else(|p| p.into_inner())
5429        .take()
5430    {
5431        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5432            module_id: module_id.to_string(),
5433            reason: reason.to_string(),
5434        }));
5435    }
5436}
5437
5438fn schedule_respawn(
5439    runtime: &SupervisorRuntimeConfig,
5440    snapshot: &SharedSnapshot,
5441    module_id: &str,
5442    delay: Duration,
5443    kind: RespawnKind,
5444) -> Result<(), SuperviseError> {
5445    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5446    update_snapshot(snapshot, Some(module_id), |state| {
5447        state.respawn_pending = true
5448    })?;
5449    *runtime
5450        .scheduled_respawn
5451        .lock()
5452        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5453        deadline: Instant::now() + delay,
5454        kind,
5455    });
5456    Ok(())
5457}
5458
5459fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5460    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5461        state.health.last_action = Some(action);
5462        state.health.last_action_ms = Some(now_ms);
5463    });
5464}
5465
5466fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5467    match status {
5468        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5469        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5470        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5471    }
5472}
5473
5474/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5475/// returned to every `supervisor.list` and `supervisor.health` caller.
5476///
5477/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5478/// path: that request exists to return a module's complete metrics object, and
5479/// `ck health <module-id>` documents it as the way to see what the cached view
5480/// truncates. The asymmetry is the feature.
5481///
5482/// So a new caller must decide which side it is on rather than assume the cap is
5483/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5484/// the truncation that path exists to avoid.
5485fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5486    let metrics = metrics?;
5487    match serde_json::to_vec(&metrics) {
5488        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5489            "truncated": true,
5490            "original_bytes": encoded.len(),
5491        })),
5492        Ok(_) | Err(_) => Some(metrics),
5493    }
5494}
5495
5496/// Spread health probes so a fleet-wide restart does not converge them.
5497///
5498/// The delay is derived from the module id and probe index rather than a random
5499/// source, so it is deterministic per module: a module keeps its own offset
5500/// across daemon restarts instead of re-rolling into a collision.
5501fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5502    if cadence.is_zero() {
5503        return Duration::ZERO;
5504    }
5505    let cadence_ms = cadence.as_millis() as u64;
5506    // This early return is REDUNDANT, deliberately, and a mutation run will show
5507    // it surviving removal. Recording why here so the next person to notice does
5508    // not have to re-derive it:
5509    //
5510    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5511    //   a zero cadence and builds the Duration from whole milliseconds, so a
5512    //   sub-millisecond cadence cannot come from config.
5513    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5514    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5515    //   -- exactly what this returns.
5516    //
5517    // Kept as a guard against a future widening of the config parser (accepting
5518    // microseconds, say), which would make the sub-millisecond case reachable.
5519    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5520    // divides by zero. Remove this and nothing changes.
5521    if cadence_ms == 0 {
5522        return cadence;
5523    }
5524    // Note that this never returns less than one cadence, including for the FIRST
5525    // probe. So a freshly registered module reports health `unknown` for a full
5526    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5527    // ready to answer.
5528    //
5529    // That is a property of the supervisor's schedule, not of any module: an
5530    // operator watching a restart sees `unknown` and cannot tell it from a module
5531    // that is slow to warm. Measured on two unrelated modules, both flipping to
5532    // `ok` between 22s and 32s after restart.
5533    //
5534    // Left as-is because spreading the first probe is what keeps a fleet-wide
5535    // restart from firing fourteen simultaneous probes into a cold machine. The
5536    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5537    // that thundering herd for a faster first reading.
5538    let jitter_span = (cadence_ms / 10).max(1);
5539    let hash = module_id.as_bytes().iter().fold(
5540        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5541        |acc, byte| {
5542            acc.wrapping_mul(1099511628211)
5543                .wrapping_add(u64::from(*byte))
5544        },
5545    );
5546    cadence + Duration::from_millis(hash % jitter_span)
5547}
5548
5549#[cfg(test)]
5550mod tests {
5551    use super::*;
5552
5553    #[test]
5554    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5555        let handle = SupervisorHandle::new();
5556        let module_id = "readded-tombstone";
5557        handle.record_rescan_removal(module_id);
5558        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5559
5560        handle.apply_identity_configuration(&ModuleSpec {
5561            module_id: module_id.to_string(),
5562            program: PathBuf::from("/test/module"),
5563            args: Vec::new(),
5564            env: Vec::new(),
5565            reserved: false,
5566            reserved_prefixes: Vec::new(),
5567            protocol: ModuleProtocol::Subc,
5568            overlap: Default::default(),
5569        });
5570
5571        assert!(
5572            handle.removal_tombstone_age_ms(module_id).is_none(),
5573            "a re-added module must not retain a stale removal tombstone"
5574        );
5575    }
5576
5577    /// What one module's owner looked like from the control plane at the
5578    /// instant after its first process was spawned.
5579    #[derive(Debug, PartialEq, Eq)]
5580    struct OwnerInSpawnWindow {
5581        module_id: String,
5582        configured: bool,
5583        on_roster: bool,
5584        admission_refusal: Option<&'static str>,
5585    }
5586
5587    /// A supervised module's process can connect, register, sync its scopes
5588    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5589    /// the module on the roster until afterward; configured supervision does
5590    /// so before spawning. In both cases the owner must
5591    /// already read as configured, so a scoped `route.open` against it is
5592    /// refused as retryable `scope_not_synced` and not as terminal
5593    /// `scope_not_live` ("will never sync").
5594    ///
5595    /// The hook runs just after each initial spawn attempt, before returning to
5596    /// the caller, so no race with a real child's registration is needed.
5597    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5598    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5599        use crate::scopes::ScopeTable;
5600        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5601
5602        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5603        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5604            module_id: module_id.to_string(),
5605            program,
5606            args: Vec::new(),
5607            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5608                .into_iter()
5609                .map(|key| (key.to_string(), dir.path().display().to_string()))
5610                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5611                .collect(),
5612            reserved: false,
5613            reserved_prefixes: Vec::new(),
5614            protocol: ModuleProtocol::Subc,
5615            overlap: Default::default(),
5616        };
5617        let live = super::terminal_history_tests::fake_aft_stub_path();
5618        let missing = dir.path().join("definitely-missing-module");
5619
5620        let handle = SupervisorHandle::new();
5621        let mut supervisor =
5622            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5623                .with_handle(handle.clone());
5624        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5625        let hook_handle = handle.clone();
5626        let hook_observed = Arc::clone(&observed);
5627        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5628            // Exactly what the control plane computes for a scoped route.open
5629            // naming this module as the owner of a scope it has not synced.
5630            let configured = hook_handle.is_configured(module_id);
5631            let selector = ScopeSelector::new(
5632                Principal::Reserved {
5633                    module_id: module_id.to_string(),
5634                },
5635                "s".to_string(),
5636                Some(1),
5637            );
5638            let carrier = Principal::Reserved {
5639                module_id: "carrier".to_string(),
5640            };
5641            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5642                .admit(&carrier, module_id, &selector, configured)
5643            {
5644                Ok(_) => None,
5645                Err(refusal) => Some(refusal.code),
5646            };
5647            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5648                module_id: module_id.to_string(),
5649                configured,
5650                on_roster: hook_handle.get(module_id).is_some(),
5651                admission_refusal,
5652            });
5653        })));
5654
5655        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5656        let configured = supervisor
5657            .supervise_configured(stub("configured", live.clone()), true)
5658            .unwrap();
5659        let with_health = supervisor
5660            .supervise_configured_with_health(
5661                stub("with-health", live.clone()),
5662                true,
5663                HealthConfig::default(),
5664                None,
5665                RestartPolicy::default(),
5666            )
5667            .unwrap();
5668        // The failed-spawn path still puts the module on the roster (as
5669        // failed), so it is configured throughout.
5670        let failed = supervisor
5671            .supervise_configured_with_health(
5672                stub("failed-spawn", missing.clone()),
5673                true,
5674                HealthConfig::default(),
5675                None,
5676                RestartPolicy::default(),
5677            )
5678            .unwrap();
5679        // A failed plain `spawn` puts nothing on the roster, so its mark is
5680        // taken back once the spawn has failed.
5681        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5682
5683        let expected = [
5684            "plain",
5685            "configured",
5686            "with-health",
5687            "failed-spawn",
5688            "spawn-error",
5689        ]
5690        .into_iter()
5691        .map(|module_id| OwnerInSpawnWindow {
5692            module_id: module_id.to_string(),
5693            configured: true,
5694            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5695            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5696        })
5697        .collect::<Vec<_>>();
5698        assert_eq!(*observed.lock().unwrap(), expected);
5699
5700        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5701            assert!(
5702                handle.get(module_id).is_some(),
5703                "{module_id} is on the roster"
5704            );
5705            assert!(
5706                handle.is_configured(module_id),
5707                "{module_id} stays configured"
5708            );
5709        }
5710        assert!(handle.get("spawn-error").is_none());
5711        assert!(
5712            !handle.is_configured("spawn-error"),
5713            "a plain spawn that failed must not leave its module marked configured"
5714        );
5715
5716        // Leaving the roster clears the mark with it.
5717        handle.retire("failed-spawn");
5718        assert!(!handle.is_configured("failed-spawn"));
5719
5720        for module in [plain, configured, with_health] {
5721            module.stop().await.unwrap();
5722        }
5723        drop(failed);
5724    }
5725
5726    /// A configured module reads as starting until the supervisor has admitted
5727    /// its first process (exec confirmed, identity durably recorded), even
5728    /// though that process is already running by then and can register over
5729    /// HELLO, which is what lists it in catalog.list. So a client that needs
5730    /// the supervisor state must wait for that state; seeing the module in
5731    /// the catalog does not mean supervisor.list already says running.
5732    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5733    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5734        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5735        let spec = ModuleSpec {
5736            module_id: "spawned-not-admitted".to_string(),
5737            program: super::terminal_history_tests::fake_aft_stub_path(),
5738            args: Vec::new(),
5739            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5740                .into_iter()
5741                .map(|key| (key.to_string(), dir.path().display().to_string()))
5742                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5743                .collect(),
5744            reserved: false,
5745            reserved_prefixes: Vec::new(),
5746            protocol: ModuleProtocol::Subc,
5747            overlap: Default::default(),
5748        };
5749        let handle = SupervisorHandle::new();
5750        let mut supervisor =
5751            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5752                .with_handle(handle.clone());
5753        let observed = Arc::new(Mutex::new(None));
5754        let hook_handle = handle.clone();
5755        let hook_observed = Arc::clone(&observed);
5756        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5757            let state = hook_handle
5758                .get(module_id)
5759                .expect("a configured module is on the roster before it is spawned")
5760                .state()
5761                .unwrap();
5762            *hook_observed.lock().unwrap() = Some(state);
5763        })));
5764
5765        let module = supervisor.supervise_configured(spec, true).unwrap();
5766        assert_eq!(
5767            *observed.lock().unwrap(),
5768            Some(ModuleState::Starting),
5769            "between the spawn and its admission the supervisor still reports starting"
5770        );
5771        assert_eq!(module.state().unwrap(), ModuleState::Running);
5772        module.stop().await.unwrap();
5773    }
5774
5775    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5776        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5777        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5778            snapshot.process_alive = true;
5779            snapshot.pid = Some(41);
5780            snapshot.spawned_at_ms = Some(42);
5781            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5782            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5783            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5784                device: 43,
5785                inode: 44,
5786                #[cfg(windows)]
5787                windows_image: None,
5788            });
5789        })
5790        .unwrap();
5791        snapshot
5792    }
5793
5794    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5795        let snapshot = lock_snapshot(snapshot).unwrap();
5796        assert!(!snapshot.process_alive);
5797        assert_eq!(snapshot.pid, None);
5798        assert_eq!(snapshot.spawned_at_ms, None);
5799        assert_eq!(snapshot.spawned_from, None);
5800        assert_eq!(snapshot.launch_spec_digest, None);
5801        assert_eq!(snapshot.spawned_file_identity, None);
5802    }
5803
5804    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5805    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5806        let supervisor =
5807            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5808        let mut runtime = supervisor.runtime_config();
5809        runtime.test_seed_stale_facts_before_enable_spawn = true;
5810        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5811        let mut child = None;
5812        let spec = ModuleSpec {
5813            module_id: "failed-enable-clears-facts".to_string(),
5814            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5815            args: Vec::new(),
5816            env: Vec::new(),
5817            reserved: false,
5818            reserved_prefixes: Vec::new(),
5819            protocol: ModuleProtocol::Subc,
5820            overlap: Default::default(),
5821        };
5822
5823        let result = set_child_enabled(
5824            &spec,
5825            &runtime,
5826            &supervisor.registry,
5827            &supervisor.process_liveness,
5828            &snapshot,
5829            &mut child,
5830            true,
5831        )
5832        .await;
5833
5834        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5835        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5836        assert_snapshot_process_facts_cleared(&snapshot);
5837    }
5838
5839    #[tokio::test]
5840    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5841        let supervisor =
5842            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5843        let runtime = supervisor.runtime_config();
5844        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5845            ModuleState::Restarting,
5846            true,
5847        )));
5848        let spec = ModuleSpec {
5849            module_id: "start-stranded-restarting".to_string(),
5850            program: super::terminal_history_tests::fake_aft_stub_path(),
5851            args: Vec::new(),
5852            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5853            reserved: false,
5854            reserved_prefixes: Vec::new(),
5855            protocol: ModuleProtocol::None,
5856            overlap: Default::default(),
5857        };
5858        let mut child = None;
5859        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5860        assert!(!super::set_child_enabled(
5861            &spec,
5862            &runtime,
5863            &Registry::default(),
5864            &supervisor.process_liveness,
5865            &snapshot,
5866            &mut child,
5867            true
5868        )
5869        .await
5870        .unwrap());
5871        assert!(child.is_none());
5872        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5873        assert!(super::set_child_enabled(
5874            &spec,
5875            &runtime,
5876            &Registry::default(),
5877            &supervisor.process_liveness,
5878            &snapshot,
5879            &mut child,
5880            true
5881        )
5882        .await
5883        .unwrap());
5884        assert_eq!(
5885            lock_snapshot(&snapshot).unwrap().state,
5886            ModuleState::Running
5887        );
5888        let mut child = child.unwrap();
5889        child.start_kill().unwrap();
5890        child.wait().await.unwrap();
5891    }
5892
5893    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5894    async fn failed_reload_spawn_clears_current_process_facts() {
5895        let supervisor =
5896            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5897        let mut runtime = supervisor.runtime_config();
5898        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5899        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5900        let mut child = None;
5901        let spec = ModuleSpec {
5902            module_id: "failed-reload-clears-facts".to_string(),
5903            program: PathBuf::from("/unused/failed-reload-module"),
5904            args: Vec::new(),
5905            env: Vec::new(),
5906            reserved: false,
5907            reserved_prefixes: Vec::new(),
5908            protocol: ModuleProtocol::Subc,
5909            overlap: Default::default(),
5910        };
5911
5912        let result = handle_reload_spawn_failure(
5913            &spec,
5914            &runtime,
5915            &supervisor.process_liveness,
5916            &snapshot,
5917            &mut child,
5918            "forced reload spawn failure".to_string(),
5919        )
5920        .await;
5921
5922        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5923        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5924        assert_snapshot_process_facts_cleared(&snapshot);
5925    }
5926
5927    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5928    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5929        let supervisor =
5930            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5931        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5932        let module = supervisor.supervised_module(
5933            ModuleSpec {
5934                module_id: "drop-clears-facts".to_string(),
5935                program: PathBuf::from("/unused/drop-module"),
5936                args: Vec::new(),
5937                env: Vec::new(),
5938                reserved: false,
5939                reserved_prefixes: Vec::new(),
5940                protocol: ModuleProtocol::Subc,
5941                overlap: Default::default(),
5942            },
5943            supervisor.runtime_config(),
5944            Arc::clone(&snapshot),
5945            None,
5946        );
5947        assert!(!module
5948            .inner
5949            .monitor
5950            .lock()
5951            .unwrap()
5952            .as_ref()
5953            .unwrap()
5954            .is_finished());
5955
5956        drop(module);
5957
5958        assert_eq!(
5959            lock_snapshot(&snapshot).unwrap().state,
5960            ModuleState::Stopped
5961        );
5962        assert_snapshot_process_facts_cleared(&snapshot);
5963    }
5964
5965    #[cfg(unix)]
5966    #[tokio::test]
5967    async fn rescan_preserves_running_protocol_until_respawn() {
5968        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5969        let initial = ModuleSpec {
5970            module_id: "rescan-protocol".into(),
5971            program: PathBuf::from("/bin/sleep"),
5972            args: vec!["60".into()],
5973            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5974                .into_iter()
5975                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5976                .collect(),
5977            reserved: false,
5978            reserved_prefixes: vec![],
5979            protocol: ModuleProtocol::None,
5980            overlap: Default::default(),
5981        };
5982        let supervisor =
5983            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5984        let module = supervisor.spawn(initial.clone()).unwrap();
5985        assert!(module.status().unwrap().live);
5986        let mut next = initial;
5987        next.protocol = ModuleProtocol::Subc;
5988        module
5989            .update_configuration(next.clone(), HealthConfig::default(), None)
5990            .await
5991            .unwrap();
5992        assert!(
5993            module.status().unwrap().live,
5994            "rescan must not require HELLO from the old non-wire process"
5995        );
5996        let runtime = supervisor.runtime_config();
5997        let action = on_child_exit(
5998            &next,
5999            RestartPolicy::default(),
6000            &supervisor.registry,
6001            &module.inner.snapshot,
6002            &runtime.terminal_ring,
6003            &runtime.spawn_events,
6004            &runtime.child_roster,
6005            ExitReport {
6006                kind: ExitKind::Clean,
6007                code: Some(0),
6008                signal: None,
6009                at_ms: unix_ms_now(),
6010            },
6011        )
6012        .await;
6013        assert!(
6014            matches!(action, NextAction::Restart { .. }),
6015            "the old non-wire process's clean exit must restart"
6016        );
6017        module.drain().await.unwrap();
6018    }
6019
6020    #[cfg(unix)]
6021    #[tokio::test]
6022    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6023        use std::os::unix::fs::PermissionsExt;
6024        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6025        let script = dir.join("module.sh");
6026        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6027        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6028        let record_path = dir.join("live-children.json");
6029        let supervisor =
6030            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6031                .with_live_children_record(&record_path);
6032        for (program, args) in [
6033            (PathBuf::from("sleep"), vec!["60".into()]),
6034            (script, vec![]),
6035        ] {
6036            let spec = ModuleSpec {
6037                module_id: "image-identity".into(),
6038                program: program.clone(),
6039                args,
6040                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6041                    .into_iter()
6042                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6043                    .collect(),
6044                reserved: false,
6045                reserved_prefixes: vec![],
6046                protocol: ModuleProtocol::None,
6047                overlap: Default::default(),
6048            };
6049            let module = supervisor.spawn(spec).unwrap();
6050            #[cfg(target_os = "macos")]
6051            {
6052                // SETEXEC confirmation is asynchronous; the orphan record must
6053                // identify the final image, never the intermediate trampoline.
6054                let deadline = Instant::now() + Duration::from_secs(5);
6055                while crate::live_children::read_record(&record_path)
6056                    .unwrap()
6057                    .iter()
6058                    .all(|entry| entry.executable.is_none())
6059                {
6060                    assert!(Instant::now() < deadline, "module image was not confirmed");
6061                    tokio::time::sleep(Duration::from_millis(5)).await;
6062                }
6063            }
6064            let entry = crate::live_children::read_record(&record_path)
6065                .unwrap()
6066                .pop()
6067                .unwrap();
6068            let observed = subc_os::Process::open(entry.pid)
6069                .unwrap()
6070                .unwrap()
6071                .observe()
6072                .unwrap();
6073            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6074            module.drain().await.unwrap();
6075            assert_eq!(
6076                verdict,
6077                crate::live_children::IdentityVerdict::Matches,
6078                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6079            );
6080        }
6081    }
6082
6083    #[cfg(unix)]
6084    fn http_fixture(
6085        dir: &std::path::Path,
6086        url: &str,
6087        threshold: u32,
6088    ) -> crate::daemon_config::ConfiguredModule {
6089        let path = dir.join("subc.jsonc");
6090        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6091            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6092            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6093            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6094            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6095        }}}).to_string()).unwrap();
6096        crate::daemon_config::load(&path)
6097            .unwrap()
6098            .unwrap()
6099            .modules
6100            .pop()
6101            .unwrap()
6102    }
6103
6104    #[cfg(unix)]
6105    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6106        timeout(Duration::from_secs(5), async {
6107            loop {
6108                if module.status().unwrap().health.status == status {
6109                    break;
6110                }
6111                sleep(Duration::from_millis(5)).await;
6112            }
6113        })
6114        .await
6115        .unwrap_or_else(|_| {
6116            panic!(
6117                "expected {status:?}, got {:?}",
6118                module.status().unwrap().health
6119            )
6120        });
6121    }
6122
6123    #[cfg(unix)]
6124    #[tokio::test]
6125    async fn http_health_status_flips_ok_failing_ok() {
6126        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6127        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6128        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6129        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6130        let serving_status = status.clone();
6131        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6132        let server = tokio::spawn(async move {
6133            loop {
6134                let (mut stream, _) = listener.accept().await.unwrap();
6135                let mut request = [0u8; 2048];
6136                let count = stream.read(&mut request).await.unwrap();
6137                assert!(count > 0, "a probe must send an HTTP request");
6138                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6139                let body = if code == 200 {
6140                    "ready"
6141                } else {
6142                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6143                };
6144                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6145                let _ = stream.write_all(response.as_bytes()).await;
6146            }
6147        });
6148        let configured = http_fixture(&dir, &url, 1000);
6149        let module =
6150            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6151                .supervise_configured_with_health(
6152                    configured.module_spec(),
6153                    true,
6154                    configured.health,
6155                    configured.drain_timeout_ms,
6156                    configured.restart,
6157                )
6158                .unwrap();
6159        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6160        status.store(503, std::sync::atomic::Ordering::SeqCst);
6161        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6162        assert!(module
6163            .status()
6164            .unwrap()
6165            .health
6166            .detail
6167            .unwrap()
6168            .contains("scratch failure"));
6169        status.store(200, std::sync::atomic::Ordering::SeqCst);
6170        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6171        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6172        let before = module.status().unwrap();
6173        let (spec, mut health) = module.configuration().unwrap();
6174        health.http = None;
6175        module
6176            .update_configuration(spec.clone(), health.clone(), Some(10))
6177            .await
6178            .unwrap();
6179        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6180        health.http = Some(url);
6181        module
6182            .update_configuration(spec, health, Some(10))
6183            .await
6184            .unwrap();
6185        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6186        assert_eq!(
6187            module.status().unwrap().pid,
6188            before.pid,
6189            "changing a probe must apply live, not restart its process"
6190        );
6191        let (spec, mut health) = module.configuration().unwrap();
6192        health.failure_threshold = 2;
6193        module
6194            .update_configuration(spec, health, Some(10))
6195            .await
6196            .unwrap();
6197        status.store(503, std::sync::atomic::Ordering::SeqCst);
6198        timeout(Duration::from_secs(5), async {
6199            while module.status().unwrap().spawn_generation == before.spawn_generation {
6200                sleep(Duration::from_millis(5)).await;
6201            }
6202        })
6203        .await
6204        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6205        module.drain().await.unwrap();
6206        server.abort();
6207    }
6208
6209    #[cfg(unix)]
6210    #[tokio::test]
6211    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6212        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6213        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6214        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6215        drop(unused);
6216        let configured = http_fixture(&dir, &url, 2);
6217        let module =
6218            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6219                .supervise_configured_with_health(
6220                    configured.module_spec(),
6221                    true,
6222                    configured.health,
6223                    configured.drain_timeout_ms,
6224                    configured.restart,
6225                )
6226                .unwrap();
6227        let before = module.status().unwrap().spawn_generation;
6228        timeout(Duration::from_secs(5), async {
6229            loop {
6230                let status = module.status().unwrap();
6231                if status.spawn_generation > before {
6232                    assert!(status.lifetime_restarts > 0);
6233                    break;
6234                }
6235                sleep(Duration::from_millis(5)).await;
6236            }
6237        })
6238        .await
6239        .expect("sustained HTTP refusal must trigger the health restart policy");
6240        module.drain().await.unwrap();
6241    }
6242
6243    #[cfg(unix)]
6244    #[tokio::test]
6245    async fn http_health_timeout_honours_deadline() {
6246        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6247        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6248        let server = tokio::spawn(async move {
6249            let _held = listener.accept().await.unwrap();
6250            std::future::pending::<()>().await;
6251        });
6252        let error = timeout(
6253            Duration::from_secs(1),
6254            probe_http_health(&url, Duration::from_millis(10)),
6255        )
6256        .await
6257        .expect("the probe must enforce its own deadline")
6258        .unwrap_err();
6259        server.abort();
6260        assert!(error.to_string().contains("timed out"));
6261    }
6262
6263    #[cfg(unix)]
6264    #[tokio::test]
6265    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6266        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6267        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6268        let url = format!(
6269            "http://localhost:{}/healthz",
6270            listener.local_addr().unwrap().port()
6271        );
6272        let server = tokio::spawn(async move {
6273            let (mut stream, _) = listener.accept().await.unwrap();
6274            let mut request = [0u8; 2048];
6275            assert!(stream.read(&mut request).await.unwrap() > 0);
6276            stream
6277                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6278                .await
6279                .unwrap();
6280        });
6281        // The deadline only bounds a hang. A probe that never tried the IPv6
6282        // address would be refused on 127.0.0.1 and fail at once, so a longer
6283        // deadline does not weaken the assertion; one second timed out under a
6284        // loaded parallel test run.
6285        assert_eq!(
6286            probe_http_health(&url, Duration::from_secs(10))
6287                .await
6288                .unwrap()
6289                .status,
6290            HealthStatus::Ok
6291        );
6292        server.await.unwrap();
6293    }
6294
6295    #[cfg(unix)]
6296    #[tokio::test]
6297    async fn http_health_timeout_keeps_partial_status_and_body() {
6298        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6299        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6300        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6301        let server = tokio::spawn(async move {
6302            let (mut stream, _) = listener.accept().await.unwrap();
6303            let mut request = [0u8; 2048];
6304            assert!(stream.read(&mut request).await.unwrap() > 0);
6305            stream
6306                .write_all(
6307                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6308                )
6309                .await
6310                .unwrap();
6311            std::future::pending::<()>().await;
6312        });
6313        let error = probe_http_health(&url, Duration::from_secs(1))
6314            .await
6315            .unwrap_err()
6316            .to_string();
6317        server.abort();
6318        assert!(
6319            error.contains("timed out")
6320                && error.contains("503 Unavailable")
6321                && error.contains("partial diagnostic"),
6322            "{error}"
6323        );
6324    }
6325
6326    #[cfg(unix)]
6327    #[tokio::test]
6328    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6329        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6330        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6331        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6332        let server = tokio::spawn(async move {
6333            let (mut stream, _) = listener.accept().await.unwrap();
6334            let mut request = [0u8; 2048];
6335            assert!(stream.read(&mut request).await.unwrap() > 0);
6336            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6337            let response = format!(
6338                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6339                body.len()
6340            );
6341            stream.write_all(response.as_bytes()).await.unwrap();
6342        });
6343        let error = probe_http_health(&url, Duration::from_secs(1))
6344            .await
6345            .unwrap_err()
6346            .to_string();
6347        server.await.unwrap();
6348        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6349        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6350        assert!(!error.contains("not-in-diagnostic"));
6351    }
6352
6353    #[cfg(unix)]
6354    #[tokio::test]
6355    async fn http_health_real_nats_server_monitoring() {
6356        if std::process::Command::new("nats-server")
6357            .arg("--version")
6358            .env("XDG_DATA_HOME", std::env::temp_dir())
6359            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6360            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6361            .output()
6362            .is_err()
6363        {
6364            eprintln!(
6365                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6366            );
6367            return;
6368        }
6369        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6370        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6371        let port = monitor.local_addr().unwrap().port();
6372        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6373        let client_port = client.local_addr().unwrap().port();
6374        let config = dir.join("server.conf");
6375        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6376        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6377        configured.program = PathBuf::from("nats-server");
6378        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6379        drop(monitor);
6380        drop(client);
6381        let module =
6382            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6383                .supervise_configured_with_health(
6384                    configured.module_spec(),
6385                    true,
6386                    configured.health,
6387                    configured.drain_timeout_ms,
6388                    configured.restart,
6389                )
6390                .unwrap();
6391        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6392        module.drain().await.unwrap();
6393    }
6394
6395    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6396    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6397        let supervisor =
6398            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6399        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6400        let initial = ModuleSpec {
6401            module_id: "rescan-preserves-spawn-facts".to_string(),
6402            program: PathBuf::from("/spawned/module"),
6403            args: Vec::new(),
6404            env: Vec::new(),
6405            reserved: false,
6406            reserved_prefixes: Vec::new(),
6407            protocol: ModuleProtocol::Subc,
6408            overlap: Default::default(),
6409        };
6410        let module = supervisor.supervised_module(
6411            initial.clone(),
6412            supervisor.runtime_config(),
6413            snapshot,
6414            None,
6415        );
6416        let before = module.status().unwrap();
6417        let mut replacement = initial;
6418        replacement.program = PathBuf::from("/rescanned/replacement-module");
6419
6420        module
6421            .update_configuration(replacement, HealthConfig::default(), None)
6422            .await
6423            .unwrap();
6424
6425        let after = module.status().unwrap();
6426        assert_eq!(after.pid, before.pid);
6427        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6428        assert_eq!(after.spawned_from, before.spawned_from);
6429        drop(module);
6430    }
6431}
6432
6433fn unix_ms_now() -> u64 {
6434    SystemTime::now()
6435        .duration_since(UNIX_EPOCH)
6436        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6437        .unwrap_or(0)
6438}
6439
6440async fn supervise_loop(
6441    mut spec: ModuleSpec,
6442    mut runtime: SupervisorRuntimeConfig,
6443    registry: Arc<Registry>,
6444    process_liveness: Arc<SupervisorProcessLiveness>,
6445    snapshot: SharedSnapshot,
6446    mut child: Option<SupervisedChild>,
6447    mut commands: mpsc::Receiver<SupervisorCommand>,
6448) {
6449    let mut health_probe = HealthProbeRuntime::default();
6450    // Registry writes (including embedded callers) notify this module only.
6451    // Subscribe before the first refresh; watch retains changes that arrive
6452    // while commands or probes are running. No polling fallback is needed.
6453    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6454        Ok(changes) => Some(changes),
6455        Err(err) => {
6456            // A poisoned registry cannot accept further writes, so it cannot
6457            // recover via a timer. Keep exit and command handling alive.
6458            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6459            None
6460        }
6461    };
6462    // All restart backoffs run here, including health and operator requests.
6463    // While one is pending the loop serves commands, so disable or drain can
6464    // cancel the replacement without spawning a process just to stop it.
6465    let mut pending_respawn: Option<PendingRespawn> = None;
6466    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6467    // before anything else so a stop that interrupted a swap runs at once.
6468    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6469    let mut previous_instance_recheck = tokio::time::interval_at(
6470        Instant::now() + Duration::from_secs(5),
6471        Duration::from_secs(5),
6472    );
6473    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6474    loop {
6475        #[cfg(test)]
6476        {
6477            lock_snapshot(&snapshot).unwrap().actor_select = None;
6478        }
6479        #[cfg(target_os = "macos")]
6480        if let Some(active) = child.as_mut() {
6481            active.confirm_privacy_exec().await;
6482        }
6483        if let Some(scheduled) = runtime
6484            .scheduled_respawn
6485            .lock()
6486            .unwrap_or_else(|p| p.into_inner())
6487            .take()
6488        {
6489            pending_respawn = Some(scheduled);
6490        }
6491        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6492            pending_respawn = None;
6493            cancel_deferred_reload(
6494                &runtime,
6495                &spec.module_id,
6496                "respawn cancelled by a supervisor command",
6497            );
6498        }
6499        if child.is_none()
6500            && pending_respawn.is_none()
6501            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6502        {
6503            cancel_deferred_reload(
6504                &runtime,
6505                &spec.module_id,
6506                "respawn cancelled before a replacement was spawned",
6507            );
6508            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6509                state.respawn_pending = false;
6510                state.coalesced_restart_pending = false;
6511                if matches!(
6512                    state.state,
6513                    ModuleState::Restarting
6514                        | ModuleState::Starting
6515                        | ModuleState::Draining
6516                        | ModuleState::Unresponsive
6517                ) {
6518                    error!(module_id = %spec.module_id, state = ?state.state,
6519                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6520                    state.state = ModuleState::Failed;
6521                    clear_current_process_facts(state);
6522                }
6523            });
6524        }
6525        if let Some(command) = requeued.pop_front() {
6526            if !handle_supervisor_command(
6527                command,
6528                &mut spec,
6529                &mut runtime,
6530                &registry,
6531                &process_liveness,
6532                &snapshot,
6533                &mut child,
6534                &mut commands,
6535                &mut requeued,
6536            )
6537            .await
6538            {
6539                return;
6540            }
6541            if child.is_some() || !respawn_still_pending(&snapshot) {
6542                pending_respawn = None;
6543                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6544                    state.respawn_pending = false
6545                });
6546            }
6547            continue;
6548        }
6549        if child.is_some() {
6550            #[cfg(test)]
6551            {
6552                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6553            }
6554            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6555            let wake_after = health_probe.wake_after();
6556            let probe_sleep = async move {
6557                match wake_after {
6558                    Some(delay) => sleep(delay).await,
6559                    None => std::future::pending().await,
6560                }
6561            };
6562            tokio::pin!(probe_sleep);
6563            // A `protocol: "none"` child (a plain process that never registers
6564            // over the subc wire, such as nats-server) with no HTTP health
6565            // check has nothing to probe, so it waits only for its exit or a
6566            // supervisor command. A rescan that adds an HTTP check is a command.
6567            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6568            #[cfg(test)]
6569            {
6570                let registration_pending = wire_child
6571                    && registration_changes
6572                        .as_ref()
6573                        .is_some_and(|changes| changes.has_changed().unwrap());
6574                let mut state = lock_snapshot(&snapshot).unwrap();
6575                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6576                    generation: state.spawn_generation,
6577                    turn: state.actor_turns,
6578                    registered_connection: health_probe.registered_connection,
6579                    next_probe_at: health_probe.next_probe_at,
6580                    wake_after,
6581                });
6582            }
6583            let active_child = child.as_mut().expect("child checked above");
6584            tokio::select! {
6585                wait_result = active_child.wait() => {
6586                    #[cfg(test)]
6587                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6588                    // Every arm below that gives up on the CHILD must keep the
6589                    // supervision task itself alive (child = None, loop
6590                    // continues into command-serving mode). Returning here
6591                    // closes the command channel, which makes the module
6592                    // permanently unrestartable in-band: a clean child exit
6593                    // of an enabled module once wedged the fleet this way
6594                    // ('supervisor command channel is closed') and required a
6595                    // full daemon restart to recover.
6596                    let exit_report = match wait_result {
6597                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6598                        Err(err) => {
6599                            active_child.drain_stderr(&spec.module_id).await;
6600                            fail_after_wait_error(
6601                                &spec.module_id,
6602                                &snapshot,
6603                                &runtime.terminal_ring,
6604                                &runtime.spawn_events,
6605                                format!("failed to wait for supervised module: {err}"),
6606                            );
6607                            untrack_if_registration_released(
6608                                &process_liveness,
6609                                &registry,
6610                                &spec.module_id,
6611                                &snapshot,
6612                            );
6613                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6614                            child = None;
6615                            continue;
6616                        }
6617                    };
6618                    active_child.drain_stderr(&spec.module_id).await;
6619
6620                    let next = on_child_exit(
6621                        &spec,
6622                        runtime.restart_policy,
6623                        &registry,
6624                        &snapshot,
6625                        &runtime.terminal_ring,
6626                        &runtime.spawn_events,
6627                        &runtime.child_roster,
6628                        exit_report,
6629                    ).await;
6630                    runtime.start_requester = None;
6631                    runtime.pending_activity_restart = false;
6632                    // The exit is recorded, so a daemon shutdown may stop
6633                    // waiting for this child (see `SupervisedChild::wait`).
6634                    active_child.release_roster();
6635                    match next {
6636                        NextAction::Stop { registration_released } => {
6637                            if registration_released {
6638                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6639                            }
6640                            child = None;
6641                        }
6642                        NextAction::Restart { schedule } => {
6643                            let delay = schedule.map_or(
6644                                runtime.restart_policy.delay_for_restart(0),
6645                                |schedule| schedule.delay,
6646                            );
6647                            if let Some(schedule) = schedule {
6648                                log_crash_respawn(&spec.module_id, schedule);
6649                            }
6650                            // The exited child is fully recorded at this point,
6651                            // so release it and count the backoff down in the
6652                            // command-serving branch below rather than sleeping
6653                            // here: commands cannot be received from inside this
6654                            // select arm, and an operator disable or drain that
6655                            // arrives during the backoff must cancel the pending
6656                            // respawn instead of waiting for it to spawn first.
6657                            child = None;
6658                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6659                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6660                        }
6661                    }
6662                }
6663                command = commands.recv() => {
6664                    #[cfg(test)]
6665                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6666                    let Some(command) = command else {
6667                        return;
6668                    };
6669                    if !handle_supervisor_command(
6670                        command,
6671                        &mut spec,
6672                        &mut runtime,
6673                        &registry,
6674                        &process_liveness,
6675                        &snapshot,
6676                        &mut child,
6677                        &mut commands,
6678                        &mut requeued,
6679                    ).await {
6680                        return;
6681                    }
6682                }
6683                _ = async {
6684                    match registration_changes.as_mut() {
6685                        Some(changes) => { let _ = changes.changed().await; }
6686                        None => std::future::pending().await,
6687                    }
6688                }, if wire_child => {
6689                    #[cfg(test)]
6690                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6691                }
6692                _ = &mut probe_sleep => {
6693                    #[cfg(test)]
6694                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6695                    if health_probe.due() {
6696                        run_health_probe_cycle(
6697                            &spec,
6698                            &runtime,
6699                            &registry,
6700                            &process_liveness,
6701                            &snapshot,
6702                            &mut child,
6703                        ).await;
6704                        if child.is_none() {
6705                            runtime.start_requester = None;
6706                            runtime.pending_activity_restart = false;
6707                        }
6708                        if child.is_some() {
6709                            health_probe.schedule_next(&spec, runtime.health.cadence);
6710                        }
6711                    }
6712                }
6713            }
6714        } else if let Some(pending) = pending_respawn {
6715            tokio::select! {
6716                _ = sleep_until(pending.deadline) => {
6717                    pending_respawn = None;
6718                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6719                    // A command handled below while the backoff elapsed may
6720                    // have stopped the module; never respawn past an operator's
6721                    // disable or drain.
6722                    if !respawn_still_pending(&snapshot) {
6723                        continue;
6724                    }
6725                    // The daemon began shutting down during the backoff: the
6726                    // spawn would be refused anyway, and refusing it here
6727                    // leaves the module stopped instead of reporting a
6728                    // failed restart.
6729                    if runtime.child_roster.is_closed() {
6730                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6731                            state.state = ModuleState::Stopped;
6732                        });
6733                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6734                        continue;
6735                    }
6736                    if let Err(err) = release_dead_registration(
6737                        &registry,
6738                        runtime.forwarding.as_deref(),
6739                        &snapshot,
6740                        &spec.module_id,
6741                    ).await {
6742                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6743                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6744                        continue;
6745                    }
6746
6747                    if matches!(pending.kind, RespawnKind::Reload) {
6748                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6749                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6750                        if result.is_ok() && runtime.pending_activity_restart {
6751                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6752                        }
6753                        runtime.pending_activity_restart = false;
6754                        if let Some(reply) = reply { let _ = reply.send(result); }
6755                        continue;
6756                    }
6757                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6758                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6759                        Ok(next_child) => {
6760                            child = Some(next_child);
6761                            if runtime.pending_activity_restart {
6762                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6763                                runtime.pending_activity_restart = false;
6764                            }
6765                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6766                        }
6767                        Err(err) => {
6768                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6769                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6770                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6771                        }
6772                    }
6773                }
6774                command = commands.recv() => {
6775                    let Some(command) = command else {
6776                        return;
6777                    };
6778                    if !handle_supervisor_command(
6779                        command,
6780                        &mut spec,
6781                        &mut runtime,
6782                        &registry,
6783                        &process_liveness,
6784                        &snapshot,
6785                        &mut child,
6786                        &mut commands,
6787                        &mut requeued,
6788                    ).await {
6789                        return;
6790                    }
6791                    // Reconcile the pending respawn with what the command did:
6792                    // a start may already have spawned a fresh child,
6793                    // while a disable or drain moved the snapshot out of the
6794                    // state the respawn was counting down from.
6795                    if child.is_some() || !respawn_still_pending(&snapshot) {
6796                        pending_respawn = None;
6797                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6798                    }
6799                }
6800            }
6801        } else {
6802            let command = if lock_snapshot(&snapshot)
6803                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6804            {
6805                tokio::select! {
6806                    command = commands.recv() => command,
6807                    _ = previous_instance_recheck.tick() => {
6808                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6809                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6810                            if enabled && !runtime.child_roster.is_closed() {
6811                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6812                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6813                                    Ok(next_child) => child = Some(next_child),
6814                                    Err(error) => {
6815                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6816                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6817                                    }
6818                                }
6819                            } else {
6820                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6821                                    state.waiting_for_previous_pid = None;
6822                                    state.waiting_for_previous_detail = None;
6823                                });
6824                            }
6825                        }
6826                        continue;
6827                    }
6828                }
6829            } else {
6830                commands.recv().await
6831            };
6832            let Some(command) = command else {
6833                return;
6834            };
6835            if !handle_supervisor_command(
6836                command,
6837                &mut spec,
6838                &mut runtime,
6839                &registry,
6840                &process_liveness,
6841                &snapshot,
6842                &mut child,
6843                &mut commands,
6844                &mut requeued,
6845            )
6846            .await
6847            {
6848                return;
6849            }
6850        }
6851    }
6852}
6853
6854fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6855    info!(
6856        module_id,
6857        restart_in_window = schedule.restart_in_window,
6858        delay_ms = schedule.delay.as_millis() as u64,
6859        "respawning after crash"
6860    );
6861}
6862
6863/// Whether the respawn a backoff was counting down to is still wanted. A
6864/// disable or drain handled while the backoff elapsed moves the snapshot out
6865/// of `Restarting`, and the operator's stop must win over the pending respawn,
6866/// so every sleep-then-spawn path re-validates against the live snapshot
6867/// instead of assuming the state it left behind still holds.
6868fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6869    matches!(
6870        lock_snapshot(snapshot),
6871        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6872    )
6873}
6874
6875enum NextAction {
6876    Stop {
6877        registration_released: bool,
6878    },
6879    Restart {
6880        schedule: Option<CrashRestartSchedule>,
6881    },
6882}
6883
6884#[allow(clippy::too_many_arguments)]
6885async fn handle_supervisor_command(
6886    command: SupervisorCommand,
6887    spec: &mut ModuleSpec,
6888    runtime: &mut SupervisorRuntimeConfig,
6889    registry: &Arc<Registry>,
6890    process_liveness: &SupervisorProcessLiveness,
6891    snapshot: &SharedSnapshot,
6892    child: &mut Option<SupervisedChild>,
6893    commands: &mut mpsc::Receiver<SupervisorCommand>,
6894    requeued: &mut VecDeque<SupervisorCommand>,
6895) -> bool {
6896    let command =
6897        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6898            match command {
6899                SupervisorCommand::Restart { reply, .. }
6900                | SupervisorCommand::Reload { reply, .. }
6901                | SupervisorCommand::Swap { reply, .. } => {
6902                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6903                    return true;
6904                }
6905                SupervisorCommand::SetEnabled {
6906                    enabled: true,
6907                    reply,
6908                    ..
6909                } => {
6910                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6911                    return true;
6912                }
6913                other => other,
6914            }
6915        } else {
6916            command
6917        };
6918    match command {
6919        SupervisorCommand::Drain { requester, reply } => {
6920            // A plain stop runs no forwarding drain, so nothing reaches the
6921            // module over its connection before the wait: ask by signal.
6922            let result = drain_optional_child(
6923                &spec.module_id,
6924                spec.protocol,
6925                StopNotice::NotSent,
6926                registry,
6927                runtime.forwarding.as_deref(),
6928                snapshot,
6929                &runtime.terminal_ring,
6930                &runtime.spawn_events,
6931                child,
6932                runtime.drain_timeout,
6933                ModuleState::Stopped,
6934                None,
6935            )
6936            .await;
6937            let registration_released = result.is_ok();
6938            if registration_released {
6939                runtime.record_activity_transition(
6940                    "subc.module_stopped",
6941                    &spec.module_id,
6942                    requester,
6943                );
6944            }
6945            let _ = reply.send(result);
6946            if registration_released {
6947                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6948            }
6949            false
6950        }
6951        SupervisorCommand::Retire { requester, reply } => {
6952            let result = async {
6953                let stop_notice = begin_forwarding_drain_if_configured(
6954                    spec,
6955                    runtime,
6956                    registry,
6957                    snapshot,
6958                    None,
6959                    RouteCloseReason::Disable,
6960                )
6961                .await?;
6962                drain_optional_child(
6963                    &spec.module_id,
6964                    spec.protocol,
6965                    stop_notice,
6966                    registry,
6967                    runtime.forwarding.as_deref(),
6968                    snapshot,
6969                    &runtime.terminal_ring,
6970                    &runtime.spawn_events,
6971                    child,
6972                    runtime.drain_timeout,
6973                    ModuleState::Stopped,
6974                    None,
6975                )
6976                .await
6977            }
6978            .await;
6979            let registration_released = result.is_ok();
6980            if registration_released {
6981                runtime.record_activity_transition(
6982                    "subc.module_stopped",
6983                    &spec.module_id,
6984                    requester,
6985                );
6986            }
6987            let _ = reply.send(result);
6988            if registration_released {
6989                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6990            }
6991            false
6992        }
6993        SupervisorCommand::Restart {
6994            requester,
6995            drain_timeout_ms,
6996            received_at_generation,
6997            queued_at,
6998            reply,
6999        } => {
7000            // Without this line a restart that waited in the queue (behind a
7001            // health probe cycle or another command) was invisible: the log
7002            // showed only the drain timing out, minutes after the operator's call.
7003            info!(
7004                module_id = %spec.module_id,
7005                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7006                "restart command dequeued"
7007            );
7008            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7009            // caller whose own request lane rides the module being restarted: the
7010            // caller's in-flight request keeps the drain from quiescing, the drain
7011            // keeps the restart from completing, and the completion keeps the reply
7012            // from releasing the caller — so the drain always timed out and cut the
7013            // initiator with a GOODBYE, even on a healthy module. Replying once the
7014            // restart is validated lets a self-lane caller settle, which is exactly
7015            // what makes the drain succeed. Completion is observable via
7016            // supervisor.list / module status; a post-ack failure lands the module
7017            // in a visible terminal state below rather than in a reply nobody can
7018            // receive.
7019            let validation = match lock_snapshot(snapshot) {
7020                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7021                    module_id: spec.module_id.clone(),
7022                }),
7023                Ok(_) => Ok(()),
7024                Err(err) => Err(err),
7025            };
7026            let initiated = validation.is_ok();
7027            let _ = reply.send(validation);
7028            // A restart asks for a fresh process. Commands run one at a time,
7029            // so a restart queued behind another restart (two operator calls
7030            // in quick succession) is dequeued the moment the first one has
7031            // spawned its replacement -- before that process has sent HELLO.
7032            // Running it would drain and kill the process the first restart
7033            // just produced, which is the opposite of what both callers asked
7034            // for. If a process spawned after this request was received is
7035            // still supervised, the request is already satisfied. Not when the
7036            // configuration changed since that spawn: then the newer process
7037            // predates the spec this restart may exist to apply.
7038            // Generation publication can lag a child that is already serving
7039            // while its identity is recorded. Only process creation after request
7040            // receipt satisfies a restart, not publication after receipt.
7041            let satisfied_by_generation = if initiated
7042                && child
7043                    .as_ref()
7044                    .is_some_and(|child| child.spawned_at > queued_at)
7045            {
7046                lock_snapshot(snapshot).ok().and_then(|state| {
7047                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7048                })
7049            } else {
7050                None
7051            };
7052            let satisfied_by_pending = initiated
7053                && child.is_none()
7054                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7055                    // No child exists yet, so this replacement will necessarily
7056                    // be created after the queued request, unlike a published
7057                    // generation whose process may already have been serving.
7058                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7059                    if pending {
7060                        state.coalesced_restart_pending = true;
7061                    }
7062                    pending
7063                });
7064            if satisfied_by_pending {
7065                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7066            } else if let Some(generation) = satisfied_by_generation {
7067                info!(
7068                    module_id = %spec.module_id,
7069                    received_at_generation,
7070                    "restart already satisfied by generation {generation}; not restarting again"
7071                );
7072            } else if initiated {
7073                // Precedence: this restart's operator override, else the module's
7074                // configured budget (already resolved into the runtime).
7075                let drain_timeout = drain_timeout_ms
7076                    .map(Duration::from_millis)
7077                    .unwrap_or(runtime.drain_timeout);
7078                if let Err(err) = restart_child(
7079                    spec,
7080                    runtime,
7081                    registry,
7082                    process_liveness,
7083                    snapshot,
7084                    child,
7085                    drain_timeout,
7086                )
7087                .await
7088                {
7089                    warn!(
7090                        module_id = %spec.module_id,
7091                        error = %err,
7092                        "operator restart failed after initiation ack; module state carries the outcome"
7093                    );
7094                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7095                        state.state = ModuleState::Failed;
7096                        clear_current_process_facts(state);
7097                    });
7098                } else {
7099                    runtime.start_requester = requester;
7100                    runtime.pending_activity_restart = true;
7101                }
7102            }
7103            true
7104        }
7105        SupervisorCommand::Reload { requester, reply } => {
7106            let result =
7107                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7108            if result.is_ok() {
7109                runtime.start_requester = requester;
7110                runtime.pending_activity_restart = true;
7111            }
7112            if result.is_ok()
7113                && runtime
7114                    .scheduled_respawn
7115                    .lock()
7116                    .unwrap_or_else(|p| p.into_inner())
7117                    .as_ref()
7118                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7119            {
7120                *runtime
7121                    .deferred_reload_reply
7122                    .lock()
7123                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7124            } else {
7125                let _ = reply.send(result);
7126            }
7127            true
7128        }
7129        SupervisorCommand::SetEnabled {
7130            enabled,
7131            requester,
7132            reply,
7133        } => {
7134            let previous_requester = runtime.start_requester.clone();
7135            if enabled {
7136                runtime.start_requester = requester.clone();
7137            }
7138            let result = set_child_enabled(
7139                spec,
7140                runtime,
7141                registry,
7142                process_liveness,
7143                snapshot,
7144                child,
7145                enabled,
7146            )
7147            .await;
7148            if matches!(result, Ok(true)) {
7149                runtime.pending_activity_restart = false;
7150                if enabled {
7151                    runtime.start_requester = requester;
7152                } else {
7153                    runtime.start_requester = None;
7154                    runtime.record_activity_transition(
7155                        "subc.module_disabled",
7156                        &spec.module_id,
7157                        requester,
7158                    );
7159                }
7160            } else {
7161                runtime.start_requester = previous_requester;
7162            }
7163            let _ = reply.send(result);
7164            true
7165        }
7166        SupervisorCommand::UpdateConfiguration {
7167            spec: next_spec,
7168            health,
7169            drain_timeout_ms,
7170            reply,
7171        } => {
7172            if let Some(handle) = &runtime.supervisor_handle {
7173                handle.apply_identity_configuration(&next_spec);
7174            }
7175            *spec = next_spec;
7176            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7177                state.configuration_updated_since_spawn = true;
7178            });
7179            let health_changed = runtime.health != health;
7180            runtime.health = health;
7181            // Reset the cadence and old endpoint's failure streak on a live
7182            // health-policy change rather than waiting for its old deadline.
7183            if health_changed {
7184                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7185                    state.health = ModuleHealthStatus::default();
7186                });
7187            }
7188            runtime.drain_timeout = drain_timeout_ms
7189                .map(Duration::from_millis)
7190                .unwrap_or(runtime.default_drain_timeout);
7191            *runtime
7192                .effective_drain_timeout
7193                .lock()
7194                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7195            let _ = reply.send(());
7196            true
7197        }
7198        SupervisorCommand::Swap {
7199            requester,
7200            ready_timeout,
7201            reply,
7202        } => {
7203            let mut swap_runtime = runtime.clone();
7204            swap_runtime.start_requester = requester;
7205            let before = lock_snapshot(snapshot)
7206                .ok()
7207                .map(|state| state.spawn_generation);
7208            let end = swap::run_swap(
7209                spec,
7210                &swap_runtime,
7211                registry,
7212                process_liveness,
7213                snapshot,
7214                child,
7215                commands,
7216                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7217                reply,
7218            )
7219            .await;
7220            if before
7221                != lock_snapshot(snapshot)
7222                    .ok()
7223                    .map(|state| state.spawn_generation)
7224            {
7225                runtime.start_requester = swap_runtime.start_requester;
7226            }
7227            requeued.extend(end.requeue);
7228            true
7229        }
7230    }
7231}
7232
7233async fn restart_child(
7234    spec: &ModuleSpec,
7235    runtime: &SupervisorRuntimeConfig,
7236    registry: &Registry,
7237    process_liveness: &SupervisorProcessLiveness,
7238    snapshot: &SharedSnapshot,
7239    child: &mut Option<SupervisedChild>,
7240    drain_timeout: Duration,
7241) -> Result<(), SuperviseError> {
7242    // Restart cycles a running module; it must not silently start a disabled one.
7243    if !lock_snapshot(snapshot)?.enabled {
7244        return Err(SuperviseError::Disabled {
7245            module_id: spec.module_id.clone(),
7246        });
7247    }
7248    let stop_notice = begin_forwarding_drain_with_timeout(
7249        spec,
7250        runtime,
7251        registry,
7252        snapshot,
7253        None,
7254        RouteCloseReason::Restart,
7255        drain_timeout,
7256    )
7257    .await?;
7258
7259    if child.is_some() {
7260        drain_optional_child(
7261            &spec.module_id,
7262            spec.protocol,
7263            stop_notice,
7264            registry,
7265            runtime.forwarding.as_deref(),
7266            snapshot,
7267            &runtime.terminal_ring,
7268            &runtime.spawn_events,
7269            child,
7270            drain_timeout,
7271            ModuleState::Restarting,
7272            Some(true),
7273        )
7274        .await?;
7275    } else {
7276        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7277            state.enabled = true;
7278            state.state = ModuleState::Restarting;
7279            clear_current_process_facts(state);
7280        })?;
7281        release_dead_registration(
7282            registry,
7283            runtime.forwarding.as_deref(),
7284            snapshot,
7285            &spec.module_id,
7286        )
7287        .await?;
7288    }
7289
7290    reset_restart_count(snapshot, &spec.module_id)?;
7291    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7292    schedule_respawn(
7293        runtime,
7294        snapshot,
7295        &spec.module_id,
7296        runtime.restart_policy.backoff,
7297        RespawnKind::Spawn,
7298    )
7299}
7300
7301async fn reload_child(
7302    spec: &ModuleSpec,
7303    runtime: &SupervisorRuntimeConfig,
7304    registry: &Registry,
7305    process_liveness: &SupervisorProcessLiveness,
7306    snapshot: &SharedSnapshot,
7307    child: &mut Option<SupervisedChild>,
7308) -> Result<(), SuperviseError> {
7309    // Reload cycles a running module; it must not silently start a disabled one.
7310    if !lock_snapshot(snapshot)?.enabled {
7311        return Err(SuperviseError::Disabled {
7312            module_id: spec.module_id.clone(),
7313        });
7314    }
7315    let stop_notice = begin_forwarding_drain(
7316        spec,
7317        runtime,
7318        registry,
7319        snapshot,
7320        Some(true),
7321        RouteCloseReason::Reload,
7322    )
7323    .await?;
7324
7325    if child.is_some() {
7326        drain_optional_child(
7327            &spec.module_id,
7328            spec.protocol,
7329            stop_notice,
7330            registry,
7331            runtime.forwarding.as_deref(),
7332            snapshot,
7333            &runtime.terminal_ring,
7334            &runtime.spawn_events,
7335            child,
7336            runtime.drain_timeout,
7337            ModuleState::Restarting,
7338            Some(true),
7339        )
7340        .await?;
7341    } else {
7342        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7343            state.enabled = true;
7344            state.state = ModuleState::Restarting;
7345            clear_current_process_facts(state);
7346        })?;
7347        release_dead_registration(
7348            registry,
7349            runtime.forwarding.as_deref(),
7350            snapshot,
7351            &spec.module_id,
7352        )
7353        .await?;
7354    }
7355
7356    reset_restart_count(snapshot, &spec.module_id)?;
7357    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7358    schedule_respawn(
7359        runtime,
7360        snapshot,
7361        &spec.module_id,
7362        runtime.restart_policy.backoff,
7363        RespawnKind::Reload,
7364    )
7365}
7366
7367async fn finish_reload_child(
7368    spec: &ModuleSpec,
7369    runtime: &SupervisorRuntimeConfig,
7370    registry: &Registry,
7371    process_liveness: &SupervisorProcessLiveness,
7372    snapshot: &SharedSnapshot,
7373    child: &mut Option<SupervisedChild>,
7374) -> Result<(), SuperviseError> {
7375    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7376    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7377        Ok(next_child) => next_child,
7378        Err(err) => {
7379            return handle_reload_spawn_failure(
7380                spec,
7381                runtime,
7382                process_liveness,
7383                snapshot,
7384                child,
7385                format!("new child failed to spawn: {err}"),
7386            )
7387            .await;
7388        }
7389    };
7390    *child = Some(next_child);
7391
7392    let wait_outcome = {
7393        let active_child = child.as_mut().expect("new reload child was just stored");
7394        wait_for_registration_after_reload(
7395            registry,
7396            &spec.module_id,
7397            snapshot,
7398            active_child,
7399            REGISTRY_RELEASE_TIMEOUT,
7400        )
7401        .await?
7402    };
7403
7404    match wait_outcome {
7405        RegistrationWaitOutcome::Registered => {
7406            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7407            Ok(())
7408        }
7409        RegistrationWaitOutcome::Exited(exit_report) => {
7410            if let Some(active_child) = child.as_mut() {
7411                active_child.drain_stderr(&spec.module_id).await;
7412            }
7413            // Keep the reaped child's roster guard until its terminal is written.
7414            // Shutdown waits on that guard, not on the child Option used for respawn.
7415            let mut exited_child = child.take().expect("exited reload child is still stored");
7416            #[cfg(test)]
7417            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7418                gate.reached.notify_one();
7419                gate.resume.notified().await;
7420            }
7421            let result = handle_reload_child_registration_failure(
7422                spec,
7423                runtime,
7424                registry,
7425                process_liveness,
7426                snapshot,
7427                child,
7428                ReloadRegistrationFailure {
7429                    exit_report: registration_failure_exit_report(exit_report),
7430                    reason: exited_child
7431                        .spawn_failure
7432                        .clone()
7433                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7434                },
7435            )
7436            .await;
7437            exited_child.release_roster();
7438            result
7439        }
7440        RegistrationWaitOutcome::TimedOut => {
7441            let mut timed_out_child = child
7442                .take()
7443                .expect("timed-out reload child is still running");
7444            timed_out_child
7445                .start_kill()
7446                .map_err(|source| SuperviseError::Kill {
7447                    module_id: spec.module_id.clone(),
7448                    source,
7449                })?;
7450            let status = timed_out_child
7451                .wait()
7452                .await
7453                .map_err(|source| SuperviseError::Wait {
7454                    module_id: spec.module_id.clone(),
7455                    source,
7456                })?;
7457            timed_out_child.drain_stderr(&spec.module_id).await;
7458            handle_reload_child_registration_failure(
7459                spec,
7460                runtime,
7461                registry,
7462                process_liveness,
7463                snapshot,
7464                child,
7465                ReloadRegistrationFailure {
7466                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7467                        snapshot,
7468                        &timed_out_child,
7469                        &status,
7470                    )),
7471                    reason: format!(
7472                        "new child did not register within {:?}",
7473                        REGISTRY_RELEASE_TIMEOUT
7474                    ),
7475                },
7476            )
7477            .await
7478        }
7479    }
7480}
7481
7482async fn set_child_enabled(
7483    spec: &ModuleSpec,
7484    runtime: &SupervisorRuntimeConfig,
7485    registry: &Registry,
7486    process_liveness: &SupervisorProcessLiveness,
7487    snapshot: &SharedSnapshot,
7488    child: &mut Option<SupervisedChild>,
7489    enabled: bool,
7490) -> Result<bool, SuperviseError> {
7491    let (current_enabled, current_state, respawn_pending) = {
7492        let state = lock_snapshot(snapshot)?;
7493        (state.enabled, state.state, state.respawn_pending)
7494    };
7495    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7496    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7497    // clean (Stopped) has no live process and no other in-band recovery — the
7498    // operator's start is the explicit recovery act and resets the budget. Without
7499    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7500    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7501    // the one providing every agent's shell.
7502    let revive_terminal = enabled
7503        && current_enabled
7504        && child.is_none()
7505        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7506            || (current_state == ModuleState::Restarting && !respawn_pending));
7507    if current_enabled == enabled && !revive_terminal {
7508        return Ok(false);
7509    }
7510
7511    if enabled {
7512        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7513            state.enabled = true;
7514            state.state = ModuleState::Starting;
7515            clear_current_process_facts(state);
7516        })?;
7517        #[cfg(test)]
7518        if runtime.test_seed_stale_facts_before_enable_spawn {
7519            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7520                state.process_alive = true;
7521                state.pid = Some(41);
7522                state.spawned_at_ms = Some(42);
7523                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7524                state.spawned_file_identity = Some(SpawnedFileIdentity {
7525                    device: 43,
7526                    inode: 44,
7527                    #[cfg(windows)]
7528                    windows_image: None,
7529                });
7530            })?;
7531        }
7532        release_dead_registration(
7533            registry,
7534            runtime.forwarding.as_deref(),
7535            snapshot,
7536            &spec.module_id,
7537        )
7538        .await?;
7539        reset_restart_count(snapshot, &spec.module_id)?;
7540        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7541        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7542            Ok(next_child) => next_child,
7543            Err(err) => {
7544                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7545                    state.state = ModuleState::Failed;
7546                    clear_current_process_facts(state);
7547                }) {
7548                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7549                }
7550                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7551                return Err(err);
7552            }
7553        };
7554        *child = Some(next_child);
7555        debug!(module_id = %spec.module_id, "supervised module enabled");
7556        Ok(true)
7557    } else {
7558        let stop_notice = begin_forwarding_drain_if_configured(
7559            spec,
7560            runtime,
7561            registry,
7562            snapshot,
7563            Some(false),
7564            RouteCloseReason::Disable,
7565        )
7566        .await?;
7567        drain_optional_child(
7568            &spec.module_id,
7569            spec.protocol,
7570            stop_notice,
7571            registry,
7572            runtime.forwarding.as_deref(),
7573            snapshot,
7574            &runtime.terminal_ring,
7575            &runtime.spawn_events,
7576            child,
7577            runtime.drain_timeout,
7578            ModuleState::Disabled,
7579            Some(false),
7580        )
7581        .await?;
7582        debug!(module_id = %spec.module_id, "supervised module disabled");
7583        Ok(true)
7584    }
7585}
7586
7587#[allow(clippy::too_many_arguments)]
7588async fn on_child_exit(
7589    spec: &ModuleSpec,
7590    policy: RestartPolicy,
7591    registry: &Registry,
7592    snapshot: &SharedSnapshot,
7593    terminal_ring: &Arc<Mutex<TerminalRing>>,
7594    spawn_events: &SpawnEventFeed,
7595    roster: &ChildRoster,
7596    exit_report: ExitReport,
7597) -> NextAction {
7598    // Once the daemon has begun shutting down, no exit is a crash to recover
7599    // from: the module is exiting because the daemon is going away (EOF on its
7600    // connection, or a service manager signalling the whole cgroup). Record it
7601    // as such and never schedule a respawn, which would only start a process
7602    // for the shutdown to end again.
7603    if roster.is_closed() {
7604        return on_child_exit_during_daemon_shutdown(
7605            spec,
7606            registry,
7607            snapshot,
7608            terminal_ring,
7609            spawn_events,
7610            exit_report,
7611        )
7612        .await;
7613    }
7614    // Every stop the supervisor itself asks for (operator stop, disable,
7615    // restart, reload, swap, a health restart, a drain that runs out of budget)
7616    // takes the child out of the supervise loop and reaps it in
7617    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7618    // that reaches this point was not requested by the daemon.
7619    //
7620    // For a subc-wire module a clean exit is still a stop: those modules are
7621    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7622    // crash, and exiting 0 is a deliberate choice the module made. A
7623    // `protocol: "none"` module is a stock program we cannot change, and many
7624    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7625    // stop would leave the module down for good after any stray signal, so it
7626    // goes through the crash path instead: it spends restart budget, respawns
7627    // with the crash backoff, and ends `failed` when the budget runs out.
7628    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7629        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7630    match exit_report.kind {
7631        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7632            info!(
7633                module_id = %spec.module_id,
7634                exit_code = ?exit_report.code,
7635                exit_signal = ?exit_report.signal,
7636                "supervised module exited cleanly"
7637            );
7638            if let Err(err) = publish_after_terminal_record(
7639                &spec.module_id,
7640                || {
7641                    record_terminal(
7642                        &spec.module_id,
7643                        terminal_ring,
7644                        spawn_events,
7645                        &exit_report,
7646                        TerminalDisposition::Stopped,
7647                    )
7648                },
7649                || {
7650                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7651                        state.state = ModuleState::Stopped;
7652                        clear_current_process_facts(state);
7653                        state.last_exit = Some(exit_report.clone());
7654                    })
7655                },
7656            ) {
7657                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7658            }
7659            let registration_released = match wait_for_registration_release(
7660                registry,
7661                &spec.module_id,
7662                REGISTRY_RELEASE_TIMEOUT,
7663            )
7664            .await
7665            {
7666                Ok(()) => true,
7667                Err(err) => {
7668                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7669                    false
7670                }
7671            };
7672            NextAction::Stop {
7673                registration_released,
7674            }
7675        }
7676        ExitKind::Clean | ExitKind::Crash => {
7677            if unrequested_clean_exit_of_protocol_none {
7678                warn!(
7679                    module_id = %spec.module_id,
7680                    exit_code = ?exit_report.code,
7681                    exit_signal = ?exit_report.signal,
7682                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7683                );
7684            } else {
7685                warn!(
7686                    module_id = %spec.module_id,
7687                    exit_code = ?exit_report.code,
7688                    exit_signal = ?exit_report.signal,
7689                    "supervised module exited abnormally (crash)"
7690                );
7691            }
7692            let mut restart_schedule = None;
7693            let mut disposition = TerminalDisposition::Disabled;
7694            let mut final_state = ModuleState::Disabled;
7695            // Set only when the budget is what stopped the module, so the
7696            // terminal record says which limit was hit rather than leaving
7697            // `failed` to be read as "crashed once, badly".
7698            let mut disposition_detail = lock_snapshot(snapshot)
7699                .ok()
7700                .and_then(|mut state| state.spawn_failure.take());
7701            let now = Instant::now();
7702            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7703                if state.enabled {
7704                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7705                        // Reserving the last slot exhausts the budget for the
7706                        // replacement, not for this exit. Publish its scheduled
7707                        // recovery atomically with the reservation so connection
7708                        // cleanup sees it while terminal recording still holds
7709                        // the visible state at Running.
7710                        state.respawn_pending = true;
7711                        final_state = ModuleState::Restarting;
7712                        restart_schedule = Some(schedule);
7713                        disposition = TerminalDisposition::Restarting;
7714                    } else {
7715                        final_state = ModuleState::Failed;
7716                        disposition = TerminalDisposition::Failed;
7717                        let budget = policy.budget_exhausted_detail();
7718                        disposition_detail =
7719                            Some(disposition_detail.take().map_or_else(
7720                                || budget.clone(),
7721                                |cause| format!("{cause}; {budget}"),
7722                            ));
7723                    }
7724                } else {
7725                    disposition = TerminalDisposition::Disabled;
7726                }
7727            }) {
7728                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7729                return NextAction::Stop {
7730                    registration_released: false,
7731                };
7732            }
7733            if disposition == TerminalDisposition::Failed {
7734                // The window is in the message, not only in the fields: this line
7735                // is read in a scrollback where a bare `max_restarts=3` reads as a
7736                // lifetime cap and sends the operator looking for three crashes
7737                // that never happened together.
7738                error!(
7739                    module_id = %spec.module_id,
7740                    max_restarts = policy.max_restarts,
7741                    window_secs = policy.window.as_secs(),
7742                    "module stopped: {}",
7743                    policy.budget_exhausted_detail()
7744                );
7745            }
7746            if let Err(err) = publish_after_terminal_record(
7747                &spec.module_id,
7748                || {
7749                    record_terminal_with_detail(
7750                        &spec.module_id,
7751                        terminal_ring,
7752                        spawn_events,
7753                        &exit_report,
7754                        disposition,
7755                        disposition_detail,
7756                    )
7757                },
7758                || {
7759                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7760                        state.state = final_state;
7761                        clear_current_process_facts(state);
7762                        state.last_exit = Some(exit_report.clone());
7763                    })
7764                },
7765            ) {
7766                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7767            }
7768
7769            if let Some(schedule) = restart_schedule {
7770                NextAction::Restart {
7771                    schedule: Some(schedule),
7772                }
7773            } else {
7774                let registration_released = match wait_for_registration_release(
7775                    registry,
7776                    &spec.module_id,
7777                    REGISTRY_RELEASE_TIMEOUT,
7778                )
7779                .await
7780                {
7781                    Ok(()) => true,
7782                    Err(err) => {
7783                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7784                        false
7785                    }
7786                };
7787                NextAction::Stop {
7788                    registration_released,
7789                }
7790            }
7791        }
7792        ExitKind::DeliberateSeverance => {
7793            warn!(
7794                module_id = %spec.module_id,
7795                exit_code = ?exit_report.code,
7796                exit_signal = ?exit_report.signal,
7797                "supervised module exited after deliberate connection severance"
7798            );
7799            let should_restart = match lock_snapshot(snapshot) {
7800                Ok(state) => state.enabled,
7801                Err(err) => {
7802                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7803                    return NextAction::Stop {
7804                        registration_released: false,
7805                    };
7806                }
7807            };
7808            let final_state = if should_restart {
7809                ModuleState::Restarting
7810            } else {
7811                ModuleState::Disabled
7812            };
7813            if let Err(err) = publish_after_terminal_record(
7814                &spec.module_id,
7815                || {
7816                    record_terminal(
7817                        &spec.module_id,
7818                        terminal_ring,
7819                        spawn_events,
7820                        &exit_report,
7821                        terminal_disposition(final_state),
7822                    )
7823                },
7824                || {
7825                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7826                        state.state = final_state;
7827                        clear_current_process_facts(state);
7828                        state.last_exit = Some(exit_report.clone());
7829                        state.lifetime_restarts += 1;
7830                    })
7831                },
7832            ) {
7833                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7834                return NextAction::Stop {
7835                    registration_released: false,
7836                };
7837            }
7838
7839            if should_restart {
7840                NextAction::Restart { schedule: None }
7841            } else {
7842                let registration_released = match wait_for_registration_release(
7843                    registry,
7844                    &spec.module_id,
7845                    REGISTRY_RELEASE_TIMEOUT,
7846                )
7847                .await
7848                {
7849                    Ok(()) => true,
7850                    Err(err) => {
7851                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7852                        false
7853                    }
7854                };
7855                NextAction::Stop {
7856                    registration_released,
7857                }
7858            }
7859        }
7860    }
7861}
7862
7863async fn on_child_exit_during_daemon_shutdown(
7864    spec: &ModuleSpec,
7865    registry: &Registry,
7866    snapshot: &SharedSnapshot,
7867    terminal_ring: &Arc<Mutex<TerminalRing>>,
7868    spawn_events: &SpawnEventFeed,
7869    exit_report: ExitReport,
7870) -> NextAction {
7871    info!(
7872        module_id = %spec.module_id,
7873        exit_code = ?exit_report.code,
7874        exit_signal = ?exit_report.signal,
7875        exit_kind = ?exit_report.kind,
7876        "supervised module exited during daemon shutdown; not restarting it"
7877    );
7878    if let Err(err) = publish_after_terminal_record(
7879        &spec.module_id,
7880        || {
7881            record_terminal(
7882                &spec.module_id,
7883                terminal_ring,
7884                spawn_events,
7885                &exit_report,
7886                TerminalDisposition::DaemonShutdown,
7887            )
7888        },
7889        || {
7890            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7891                state.state = ModuleState::Stopped;
7892                clear_current_process_facts(state);
7893                state.last_exit = Some(exit_report.clone());
7894            })
7895        },
7896    ) {
7897        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7898    }
7899    let registration_released =
7900        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7901            .await
7902            .is_ok();
7903    NextAction::Stop {
7904        registration_released,
7905    }
7906}
7907
7908// Keep the status lock out of terminal I/O. Readers may observe the previous
7909// state while recording is in progress, but never a completed exit without its
7910// history. Journal errors are logged by the journal and retained in the ring;
7911// a recording panic still publishes the final state before resuming the unwind.
7912fn publish_after_terminal_record<T>(
7913    module_id: &str,
7914    record: impl FnOnce(),
7915    publish: impl FnOnce() -> T,
7916) -> T {
7917    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7918    let published = publish();
7919    if let Err(panic) = recorded {
7920        error!(
7921            module_id,
7922            "terminal recording panicked; published final state before resuming unwind"
7923        );
7924        std::panic::resume_unwind(panic);
7925    }
7926    published
7927}
7928
7929fn fail_after_wait_error(
7930    module_id: &str,
7931    snapshot: &SharedSnapshot,
7932    terminal_ring: &Arc<Mutex<TerminalRing>>,
7933    spawn_events: &SpawnEventFeed,
7934    detail: String,
7935) {
7936    let report = wait_error_exit_report();
7937    publish_after_terminal_record(
7938        module_id,
7939        || {
7940            record_terminal_with_detail(
7941                module_id,
7942                terminal_ring,
7943                spawn_events,
7944                &report,
7945                TerminalDisposition::Failed,
7946                Some(detail),
7947            )
7948        },
7949        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7950    );
7951}
7952
7953#[cfg(test)]
7954fn record_wait_error_terminal(
7955    module_id: &str,
7956    terminal_ring: &Arc<Mutex<TerminalRing>>,
7957    spawn_events: &SpawnEventFeed,
7958) {
7959    record_terminal(
7960        module_id,
7961        terminal_ring,
7962        spawn_events,
7963        &wait_error_exit_report(),
7964        TerminalDisposition::Failed,
7965    );
7966}
7967
7968fn record_terminal(
7969    module_id: &str,
7970    terminal_ring: &Arc<Mutex<TerminalRing>>,
7971    spawn_events: &SpawnEventFeed,
7972    exit_report: &ExitReport,
7973    disposition: TerminalDisposition,
7974) {
7975    record_terminal_with_detail(
7976        module_id,
7977        terminal_ring,
7978        spawn_events,
7979        exit_report,
7980        disposition,
7981        None,
7982    );
7983}
7984
7985/// The ring lock is held only to capture the read (see
7986/// `TerminalJournal::capture_read`), so this module's exits keep recording
7987/// while the journal files are read. Blocking: it reads files.
7988fn durable_terminal_history_of(
7989    terminal_ring: &Mutex<TerminalRing>,
7990    module_id: &str,
7991) -> subc_control::TerminalHistory {
7992    let read = terminal_ring
7993        .lock()
7994        .unwrap_or_else(|p| p.into_inner())
7995        .capture_durable_history();
7996    read.read(module_id)
7997}
7998
7999fn record_terminal_with_detail(
8000    module_id: &str,
8001    terminal_ring: &Arc<Mutex<TerminalRing>>,
8002    spawn_events: &SpawnEventFeed,
8003    exit_report: &ExitReport,
8004    disposition: TerminalDisposition,
8005    disposition_detail: Option<String>,
8006) {
8007    #[cfg(test)]
8008    terminal_history_tests::before_terminal_record(spawn_events);
8009    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8010    let record = TerminalRecord {
8011        exit_code: exit_report.code,
8012        exit_signal: exit_report.signal,
8013        at_ms: exit_report.at_ms,
8014        disposition,
8015        exit_kind: exit_report.kind.into(),
8016        disposition_detail,
8017    };
8018    terminal_ring
8019        .lock()
8020        .unwrap_or_else(|poisoned| poisoned.into_inner())
8021        .record_exit(module_id, record);
8022}
8023
8024fn untrack_if_registration_released(
8025    process_liveness: &SupervisorProcessLiveness,
8026    registry: &Registry,
8027    module_id: &str,
8028    snapshot: &SharedSnapshot,
8029) {
8030    match registry.get_module(module_id) {
8031        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8032        Ok(Some(_)) => {}
8033        Err(err) => {
8034            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8035        }
8036    }
8037}
8038
8039/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8040/// then apply the module's configured entries minus daemon-private capture keys.
8041///
8042/// Separated from `spawn_child` only so it can be asserted without spawning a
8043/// process — a duplicate of this logic in a test would pass while the real one
8044/// drifted, which is the defect class this function exists to avoid.
8045/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8046/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8047/// either and the argument would stop a stock binary from starting at all.
8048/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8049///
8050/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8051/// through [`apply_wire_spawn_args_for_role`].
8052#[cfg(test)]
8053fn apply_wire_spawn_args(
8054    command: &mut Command,
8055    spec: &ModuleSpec,
8056    connection_file_path: Option<&std::path::Path>,
8057    handle: Option<&SupervisorHandle>,
8058) -> Result<Option<NonceHandoff>, SuperviseError> {
8059    apply_wire_spawn_args_for_role(
8060        command,
8061        spec,
8062        connection_file_path,
8063        handle,
8064        SpawnRole::Plain,
8065    )
8066}
8067
8068/// The read end of a spawn's launch-nonce pipe, prepared by
8069/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8070/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8071/// server before spawn and authenticates its client after the child exists.
8072#[cfg(unix)]
8073type NonceHandoff = subc_os::LaunchNonceHandoff;
8074#[cfg(windows)]
8075type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8076#[cfg(not(any(unix, windows)))]
8077type NonceHandoff = std::convert::Infallible;
8078
8079/// Create the launch nonce that a new module process must present when it
8080/// registers or opens a consumer connection, including a replacement process
8081/// started alongside a still-serving module.
8082///
8083/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8084/// a separate candidate token so the still-serving incumbent and its consumers
8085/// keep their nonce. Both records are installed before the process exists, so
8086/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8087///
8088/// On Unix the nonce is delivered only through a pipe. It is written into
8089/// a pipe whose read end the child gets as descriptor 3, named by
8090/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8091/// process of the same user cannot read it with `ps eww`. That handoff is
8092/// returned rather than installed here, because installing it replaces
8093/// whatever the child has at descriptor 3 and so must be the last pre-exec
8094/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8095/// the Linux callback that writes the process into its cgroup. Only after that
8096/// does `handoff.install_last` register the descriptor-3 replacement, so it
8097/// cannot close a descriptor the cgroup callback still needs.
8098/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8099fn apply_wire_spawn_args_for_role(
8100    command: &mut Command,
8101    spec: &ModuleSpec,
8102    connection_file_path: Option<&std::path::Path>,
8103    handle: Option<&SupervisorHandle>,
8104    role: SpawnRole,
8105) -> Result<Option<NonceHandoff>, SuperviseError> {
8106    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8107    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8108    // included: a daemon started from a module's process tree inherits it,
8109    // and passing it on would point the child at a descriptor it does not
8110    // have.
8111    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8112    #[cfg(windows)]
8113    {
8114        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8115        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8116    }
8117    // Remove inherited or configured copies too: withholding must mean absent.
8118    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8119    if spec.protocol == ModuleProtocol::None {
8120        return Ok(None);
8121    }
8122    if let Some(connection_file_path) = connection_file_path {
8123        command.arg(SUBC_ARG).arg(connection_file_path);
8124    }
8125
8126    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8127    // route.open attestation. Reserved modules additionally use the same nonce
8128    // for HELLO id-squatting protection. A respawn rotates both records.
8129    let nonce = generate_launch_nonce()?;
8130    if let Some(handle) = handle {
8131        match role {
8132            SpawnRole::Plain => {
8133                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8134                if spec.reserved {
8135                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8136                }
8137            }
8138            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8139        }
8140    }
8141    #[cfg(unix)]
8142    let handoff = {
8143        let handoff =
8144            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8145                program: spec.program.clone(),
8146                source,
8147                cgroup_path: None,
8148            })?;
8149        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8150        Some(handoff)
8151    };
8152    #[cfg(windows)]
8153    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8154    #[cfg(not(any(unix, windows)))]
8155    let handoff = None;
8156    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8157    // until the daemon's per-spawn delivery records show that every running
8158    // Windows module has read its secret from the named pipe.
8159    #[cfg(not(unix))]
8160    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8161    Ok(handoff)
8162}
8163
8164fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8165    command.env_remove(CK_LOG_ENV);
8166    // The spawn role is the supervisor's to set, and only on a swap candidate
8167    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8168    // it, is what makes it absent on a plain spawn: the daemon's own
8169    // environment could carry it, and so could a spec built outside daemon
8170    // config (config refuses it as an `env` key). A module reading it on a
8171    // plain restart would pick the long swap budget and leave callers waiting.
8172    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8173    for (key, value) in &spec.env {
8174        // cortexkit-log currently exposes retention only as a Rust struct, not
8175        // environment names. These values are daemon-private sink metadata and
8176        // must never become a public child-process contract by being inherited.
8177        if matches!(
8178            key.as_str(),
8179            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8180        ) || key == SUBC_SPAWN_ROLE_ENV
8181        {
8182            continue;
8183        }
8184        command.env(key, value);
8185    }
8186}
8187
8188#[cfg(windows)]
8189fn prepare_windows_nonce_pipe(
8190    command: &mut Command,
8191    spec: &ModuleSpec,
8192    nonce: &str,
8193) -> Option<NonceHandoff> {
8194    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8195        Ok(handoff) => {
8196            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8197            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8198            // still register until every module reads the pipe. A reader without
8199            // this variable set to env returns the pipe error instead of using
8200            // an inherited or configured environment copy.
8201            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8202            Some(handoff)
8203        }
8204        Err(error) => {
8205            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8206            None
8207        }
8208    }
8209}
8210
8211#[cfg(windows)]
8212fn start_windows_nonce_pipe(
8213    handoff: Option<NonceHandoff>,
8214    pid: u32,
8215    spec: &ModuleSpec,
8216) -> Option<subc_os::LaunchNoncePipeDelivery> {
8217    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8218    // same maximum wait used for registration during a reload. Older modules
8219    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8220    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8221        Ok(delivery) => Some(delivery),
8222        Err(error) => {
8223            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8224            None
8225        }
8226    })
8227}
8228
8229fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8230    #[cfg(windows)]
8231    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8232        return Some(
8233            if snapshot
8234                .pipe_consumed
8235                .as_ref()
8236                .is_some_and(|consumed| consumed.get().is_some())
8237            {
8238                "pipe"
8239            } else {
8240                "env"
8241            }
8242            .to_owned(),
8243        );
8244    }
8245    let _ = snapshot;
8246    None
8247}
8248
8249/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8250/// of a blue/green swap.
8251#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8252enum SpawnRole {
8253    Plain,
8254    SwapCandidate,
8255}
8256
8257/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8258/// `apply_child_env` has already removed the variable for every spawn.
8259fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8260    if role == SpawnRole::SwapCandidate {
8261        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8262    }
8263}
8264
8265fn previous_instance_error(
8266    spec: &ModuleSpec,
8267    pid: u32,
8268    reason: crate::live_children::RetainedReason,
8269) -> SuperviseError {
8270    SuperviseError::Spawn {
8271        program: spec.program.clone(),
8272        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8273        cgroup_path: None,
8274    }
8275}
8276
8277fn spawn_child(
8278    spec: &ModuleSpec,
8279    connection_file_path: Option<&std::path::Path>,
8280    handle: Option<&SupervisorHandle>,
8281    ring: &Arc<Mutex<StderrRing>>,
8282    capture_logs_dir: Option<&std::path::Path>,
8283    roster: &ChildRoster,
8284    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8285) -> Result<SupervisedChild, SuperviseError> {
8286    spawn_child_in_slot(
8287        spec,
8288        connection_file_path,
8289        handle,
8290        ring,
8291        capture_logs_dir,
8292        roster,
8293        #[cfg(target_os = "linux")]
8294        cgroup_placement,
8295        SpawnRole::Plain,
8296        false,
8297    )
8298}
8299
8300/// Spawn one process of `spec` into a slot.
8301///
8302/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8303/// A swap candidate needs a different cgroup from the process it is replacing,
8304/// which is still alive: in the same cgroup the two would be one kill domain,
8305/// and killing a failed candidate could take the incumbent with it.
8306///
8307/// The stderr capture file is `<module_id>.stderr.log` for every process of
8308/// the module, whichever slot it is in, because that is the one file
8309/// `ck module logs` reads. During a swap's overlap both processes append to it;
8310/// the daemon writes whole lines, so the two interleave by line, which is also
8311/// the merged view an operator wants while a swap runs.
8312#[allow(clippy::too_many_arguments)]
8313fn spawn_child_in_slot(
8314    spec: &ModuleSpec,
8315    connection_file_path: Option<&std::path::Path>,
8316    handle: Option<&SupervisorHandle>,
8317    ring: &Arc<Mutex<StderrRing>>,
8318    capture_logs_dir: Option<&std::path::Path>,
8319    roster: &ChildRoster,
8320    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8321    role: SpawnRole,
8322    alternate_slot: bool,
8323) -> Result<SupervisedChild, SuperviseError> {
8324    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8325        return Err(previous_instance_error(spec, pid, reason));
8326    }
8327    if roster.is_closed() {
8328        return Err(SuperviseError::Spawn {
8329            program: spec.program.clone(),
8330            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8331            cgroup_path: None,
8332        });
8333    }
8334    #[cfg(target_os = "linux")]
8335    let cgroup_name = {
8336        // Slot names alone are not kill domains: a retired incumbent may still
8337        // be draining when a later enable/restart spawns into the same slot.
8338        // Decimal entropy keeps the suffix unambiguous; Placement performs
8339        // the module-id escaping and constructs the filesystem path.
8340        if cgroup_placement.is_none() {
8341            swap::cgroup_name(&spec.module_id, alternate_slot)
8342        } else {
8343            let nonce = generate_launch_nonce()?;
8344            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8345            // Leave room for byte escaping and the suffix under NAME_MAX. The
8346            // label is only for humans; the nonce identifies the kill domain.
8347            let mut end = spec.module_id.len().min(64);
8348            while !spec.module_id.is_char_boundary(end) {
8349                end -= 1;
8350            }
8351            format!(
8352                "{}_{suffix}",
8353                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8354            )
8355        }
8356    };
8357    #[cfg(not(target_os = "linux"))]
8358    let _ = alternate_slot;
8359    #[cfg(target_os = "macos")]
8360    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8361    #[cfg(windows)]
8362    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8363        program: spec.program.clone(),
8364        source,
8365        cgroup_path: None,
8366    })?;
8367    #[cfg(windows)]
8368    let mut command = Command::new(image_capture.path());
8369    #[cfg(not(any(target_os = "macos", windows)))]
8370    let mut command = Command::new(&spec.program);
8371    command.args(&spec.args);
8372    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8373    // that is the whole of the intent, so remove that one key rather than the
8374    // environment.
8375    //
8376    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8377    // and took the POSIX environment with it. Modules spawned that way had no
8378    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8379    // logging:
8380    //
8381    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8382    //     both unset it fell back to the temp dir alone and `ck` could not find
8383    //     a daemon running on the same machine from inside any module's process
8384    //     tree — reporting a path the file has never lived at, which reads as
8385    //     "the daemon did not write its file".
8386    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8387    //     the RELATIVE `.local/share`, so a module deriving its own store path
8388    //     resolved it against its own CWD. That is the store-fragmentation
8389    //     defect the daemon already refuses in config (`parse_doc` rejects a
8390    //     relative `storage.data_home`) arriving by derivation instead.
8391    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8392    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8393    //     quietly rather than erroring.
8394    //
8395    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8396    // offered one candidate under /tmp while the file sat in /run/user/1000.
8397    //
8398    // A configured module is unaffected either way: `module_spec()` puts the
8399    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8400    // wins over anything ambient.
8401    apply_child_env(&mut command, spec);
8402    apply_spawn_role(&mut command, role);
8403    let nonce_handoff =
8404        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8405
8406    #[cfg(target_os = "linux")]
8407    let cgroup_path = cgroup_placement
8408        .map(|placement| placement.module_path(&cgroup_name))
8409        .transpose()
8410        .map_err(|source| SuperviseError::Cgroup {
8411            module_id: spec.module_id.clone(),
8412            source,
8413        })?;
8414    #[cfg(not(target_os = "linux"))]
8415    let cgroup_path: Option<PathBuf> = None;
8416    #[cfg(target_os = "linux")]
8417    if let Some(path) = &cgroup_path {
8418        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8419            if let Some(placement) = cgroup_placement {
8420                remove_module_cgroup(placement, &cgroup_name);
8421            }
8422            return Err(error);
8423        }
8424    }
8425
8426    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8427        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8428        match ChildOutputSink::open(&path, capture_retention(spec)) {
8429            Ok(sink) => sink,
8430            Err(error) => {
8431                warn!(
8432                    module_id = %spec.module_id,
8433                    path = %path.display(),
8434                    error = %error,
8435                    "could not open child output capture file; forwarding to stderr"
8436                );
8437                ChildOutputSink::Stderr
8438            }
8439        }
8440    } else {
8441        ChildOutputSink::Stderr
8442    };
8443
8444    command.stdout(Stdio::piped());
8445    command.stderr(Stdio::piped());
8446    command.kill_on_drop(true);
8447    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8448    // before exec). In the daemon's group, a service manager that kills the
8449    // job's process group when the daemon exits (launchd's default) killed
8450    // every module at the same moment its control connection closed, so no
8451    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8452    // module is reached only by the daemon: the EOF it sees when its
8453    // connection closes, and the bounded stop in `child_roster` for anything
8454    // still running after that. On Linux this composes with the cgroup
8455    // placement above: that is a pre_exec write to cgroup.procs, std performs
8456    // setpgid in the child before running pre_exec callbacks, and the two
8457    // change independent process attributes.
8458    //
8459    // stdin is /dev/null because a process outside the terminal's foreground
8460    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8461    // by hand would otherwise hand down. Under a service manager stdin is
8462    // already /dev/null.
8463    #[cfg(unix)]
8464    command.process_group(0);
8465    command.stdin(Stdio::null());
8466    // The LAST pre-exec step, after the cgroup placement above: installing the
8467    // nonce at descriptor 3 replaces whatever the child had there, which could
8468    // be the descriptor an earlier step writes through.
8469    #[cfg(unix)]
8470    if let Some(handoff) = nonce_handoff {
8471        handoff.install_last(command.as_std_mut());
8472    }
8473    #[cfg(not(any(unix, windows)))]
8474    let _ = nonce_handoff;
8475
8476    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8477    // cannot run a single instruction -- and therefore cannot spawn a
8478    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8479    // other two steps and why the window matters.
8480    #[cfg(windows)]
8481    {
8482        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8483            program: spec.program.clone(),
8484            source,
8485            cgroup_path: None,
8486        })?;
8487        subc_jobobject::suspend_on_create_async(&mut command);
8488    }
8489    // Timestamp before the spawn call: on Unix the child can serve before that
8490    // call even returns. This conservative bound never labels an older child
8491    // as a replacement for a request received during process creation.
8492    let spawned_at = Instant::now();
8493    let mut child = match command.spawn() {
8494        Ok(child) => child,
8495        Err(source) => {
8496            #[cfg(target_os = "linux")]
8497            if let Some(placement) = cgroup_placement {
8498                remove_module_cgroup(placement, &cgroup_name);
8499            }
8500            return Err(SuperviseError::Spawn {
8501                program: spec.program.clone(),
8502                source,
8503                cgroup_path,
8504            });
8505        }
8506    };
8507    #[cfg(test)]
8508    spawn_record_tests::after_create(&child, spec);
8509    // The parent must close its writer now: the acknowledgement pipe reports EOF
8510    // only when every writer is gone, and the child's copy closes when the
8511    // trampoline replaces itself with the module. Command holds only an integer
8512    // in its pre_exec callback, not another writer.
8513    #[cfg(target_os = "macos")]
8514    drop(exec_ack);
8515
8516    // Bind while the child is still suspended, before job assignment resumes it.
8517    // Keeping the executable handle open across CreateProcessW denies rename
8518    // and deletion, so the file cannot be exchanged during spawn. Bind that
8519    // file ID to the process handle returned by spawn, never a reopened PID.
8520    #[cfg(windows)]
8521    let windows_image = match image_capture.bind_tokio(&child) {
8522        Ok(image) => Arc::new(image),
8523        Err(source) => {
8524            kill_and_reap_rejected_child(&mut child);
8525            return Err(SuperviseError::Spawn {
8526                program: spec.program.clone(),
8527                source,
8528                cgroup_path: None,
8529            });
8530        }
8531    };
8532    // Assign the suspended child to its module job; resume only after its
8533    // process identity has been durably recorded for orphan recovery.
8534    #[cfg(windows)]
8535    let job = contain_spawned_child(&mut child, spec)?;
8536    let spawned_at_ms = unix_ms_now();
8537    #[cfg(not(windows))]
8538    let spawned_from = spec.program.clone();
8539    #[cfg(windows)]
8540    let spawned_from = windows_image.path().to_path_buf();
8541    #[cfg(not(windows))]
8542    let spawned_file_identity = spawned_file_identity(&spawned_from);
8543    #[cfg(windows)]
8544    let spawned_file_identity = Some(SpawnedFileIdentity {
8545        device: 0,
8546        inode: 0,
8547        windows_image: Some(Arc::clone(&windows_image)),
8548    });
8549    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8550        program: spec.program.clone(),
8551        source: io::Error::other("spawned child exposed no live pid"),
8552        cgroup_path: cgroup_path.clone(),
8553    })?;
8554    #[cfg(windows)]
8555    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8556    #[cfg(not(windows))]
8557    let process_start_time = crate::provenance::process_start_time(pid);
8558    #[cfg(windows)]
8559    let process_start_time = Some(windows_image.start_time());
8560    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8561    #[cfg(all(test, target_os = "macos"))]
8562    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8563    // Record the committed kernel image, not the configured pathname: PATH
8564    // lookup and shebang interpretation can select a different executable.
8565    // Linux's posix_spawn can return before that image is installed, so the
8566    // observer also waits for the kernel's first-exec flag to clear.
8567    #[cfg(not(windows))]
8568    let recorded_image = observe_spawned_image(pid);
8569    // spawn() confirms only the first exec, into the trampoline. Never persist
8570    // the trampoline image; the asynchronous acknowledgement publishes the
8571    // module image once the trampoline has replaced itself with the module.
8572    #[cfg(target_os = "macos")]
8573    let recorded_image = if privacy_exec.is_some() {
8574        None
8575    } else {
8576        recorded_image
8577    };
8578    #[cfg(target_os = "linux")]
8579    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8580    #[cfg(not(target_os = "linux"))]
8581    let recorded_cgroup_name = None;
8582    let roster_guard = roster.admit(
8583        spec.module_id.clone(),
8584        pid,
8585        spec.protocol,
8586        process_start_time,
8587        crate::child_roster::RecordedIdentity {
8588            #[cfg(not(windows))]
8589            start_time: recorded_image.map(|image| image.start_time),
8590            #[cfg(windows)]
8591            start_time: process_start_time,
8592            #[cfg(not(windows))]
8593            executable: recorded_image
8594                .and_then(|image| image.executable)
8595                .map(crate::live_children::ExecutableIdentity::from),
8596            #[cfg(windows)]
8597            executable: Some(crate::live_children::ExecutableIdentity::from(
8598                windows_image.identity(),
8599            )),
8600            cgroup_name: recorded_cgroup_name,
8601            #[cfg(target_os = "linux")]
8602            cgroup_placement: cgroup_placement.cloned(),
8603        },
8604    );
8605    let roster_guard = match roster_guard {
8606        Ok(guard) => guard,
8607        Err(source) => {
8608            #[cfg(windows)]
8609            if let Some(job) = &job {
8610                let _ = job.terminate();
8611            }
8612            #[cfg(target_os = "linux")]
8613            kill_module_cgroup(cgroup_placement, &cgroup_name);
8614            kill_and_reap_rejected_child(&mut child);
8615            #[cfg(target_os = "linux")]
8616            if let Some(placement) = cgroup_placement {
8617                remove_module_cgroup(placement, &cgroup_name);
8618            }
8619            return Err(SuperviseError::Spawn {
8620                program: spec.program.clone(),
8621                source: io::Error::other(format!(
8622                    "could not durably record module child: {source}"
8623                )),
8624                cgroup_path,
8625            });
8626        }
8627    };
8628    // The check at the top of this function can pass just before daemon
8629    // shutdown begins, and the process is only in the roster from here on.
8630    // The shutdown stop returns as soon as it finds the roster empty, so a
8631    // process admitted after that look would outlive the daemon. The roster
8632    // is closed before the stop first reads it and admission happens under
8633    // the roster's lock, so either the stop sees this process or this check
8634    // sees the roster closed: end the process now rather than start a module
8635    // the daemon is about to stop.
8636    if roster.is_closed() {
8637        // This child was never admitted, so there is no module protocol shutdown to wait for.
8638        #[cfg(target_os = "linux")]
8639        kill_module_cgroup(cgroup_placement, &cgroup_name);
8640        if let Err(error) = child.start_kill() {
8641            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8642        }
8643        #[cfg(target_os = "linux")]
8644        if let Some(placement) = cgroup_placement {
8645            // This spawn was never admitted, so shutdown has no roster entry
8646            // to await. Do not detach its cleanup: the runtime could exit
8647            // before that task reaps the rejected child and removes its group.
8648            while matches!(child.try_wait(), Ok(None)) {
8649                std::thread::yield_now();
8650            }
8651            if matches!(
8652                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8653                subc_cgroup::KillOutcome::Killed
8654            ) {
8655                if let Ok(path) = placement.module_path(&cgroup_name) {
8656                    while std::fs::read_to_string(path.join("cgroup.events"))
8657                        .ok()
8658                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8659                    {
8660                        std::thread::yield_now();
8661                    }
8662                }
8663            }
8664            remove_module_cgroup(placement, &cgroup_name);
8665        }
8666        drop(roster_guard);
8667        return Err(SuperviseError::Spawn {
8668            program: spec.program.clone(),
8669            source: io::Error::other(
8670                "the daemon began shutting down while this process was starting; ended it",
8671            ),
8672            cgroup_path,
8673        });
8674    }
8675
8676    #[cfg(windows)]
8677    if let Err(error) = resume_suspended_child(pid, spec) {
8678        kill_and_reap_rejected_child(&mut child);
8679        return Err(error);
8680    }
8681
8682    let stdout_pump = match child.stdout.take() {
8683        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8684        None => {
8685            warn!(
8686                module_id = %spec.module_id,
8687                "spawned child exposed no stdout pipe; file capture will be incomplete"
8688            );
8689            None
8690        }
8691    };
8692    let stderr_pump = match child.stderr.take() {
8693        Some(stderr) => {
8694            let generation = ring
8695                .lock()
8696                .unwrap_or_else(|poisoned| poisoned.into_inner())
8697                .begin_process();
8698            Some(StderrPump {
8699                task: tokio::spawn(pump_stderr_to(
8700                    stderr,
8701                    Arc::clone(ring),
8702                    generation,
8703                    output_sink,
8704                )),
8705                generation,
8706            })
8707        }
8708        None => {
8709            // Spawning succeeded but the pipe did not materialise. Recording it as
8710            // uncaptured keeps the tail honest: the alternative is an empty tail
8711            // that reads as a module which printed nothing.
8712            ring.lock()
8713                .unwrap_or_else(|poisoned| poisoned.into_inner())
8714                .mark_not_captured("stderr pipe was not available on spawn");
8715            warn!(
8716                module_id = %spec.module_id,
8717                "spawned child exposed no stderr pipe; tail will be unavailable"
8718            );
8719            None
8720        }
8721    };
8722
8723    let exec = Arc::new(crate::activity::ExecOutcome::new());
8724    #[cfg(target_os = "macos")]
8725    let pending_exec = privacy_exec.is_some();
8726    #[cfg(not(target_os = "macos"))]
8727    let pending_exec = false;
8728    if !pending_exec {
8729        exec.finish(crate::activity::ExecState::CommittedExec);
8730    }
8731    Ok(SupervisedChild {
8732        child,
8733        exec: crate::activity::ExecOwner(exec),
8734        #[cfg(windows)]
8735        nonce_delivery,
8736        protocol: spec.protocol,
8737        #[cfg(target_os = "linux")]
8738        module_id: cgroup_name,
8739        #[cfg(target_os = "linux")]
8740        cgroup_placement: cgroup_placement.cloned(),
8741        #[cfg(windows)]
8742        job,
8743        stdout_pump,
8744        stderr_pump,
8745        stderr_ring: Arc::clone(ring),
8746        spawned_at,
8747        spawned_at_ms,
8748        spawned_from,
8749        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8750        spawned_file_identity,
8751        process_start_time,
8752        process_identity,
8753        pid,
8754        roster_guard: Some(roster_guard),
8755        #[cfg(target_os = "macos")]
8756        privacy_exec,
8757        #[cfg(target_os = "macos")]
8758        report_ready: Arc::new(OnceLock::new()),
8759        spawn_failure: None,
8760    })
8761}
8762
8763#[cfg(test)]
8764mod spawn_record_tests {
8765    use super::*;
8766
8767    thread_local! {
8768        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8769    }
8770
8771    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8772        LAST_PID.set(child.id());
8773        #[cfg(windows)]
8774        if let Some((_, address)) = _spec
8775            .env
8776            .iter()
8777            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8778        {
8779            use std::io::{Read, Write};
8780            let mut stream = std::net::TcpStream::connect(address).unwrap();
8781            stream
8782                .write_all(&child.id().unwrap().to_ne_bytes())
8783                .unwrap();
8784            let mut release = [0];
8785            stream.read_exact(&mut release).unwrap();
8786        }
8787    }
8788
8789    #[cfg(windows)]
8790    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8791        if spec
8792            .env
8793            .iter()
8794            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8795        {
8796            Err(io::Error::other("injected module job assignment failure"))
8797        } else {
8798            Ok(())
8799        }
8800    }
8801
8802    fn spec() -> ModuleSpec {
8803        ModuleSpec {
8804            module_id: "spawn-record-worker".into(),
8805            #[cfg(unix)]
8806            program: "/bin/sleep".into(),
8807            #[cfg(windows)]
8808            program: std::env::current_exe().unwrap(),
8809            #[cfg(unix)]
8810            args: vec!["60".into()],
8811            #[cfg(windows)]
8812            args: vec![
8813                "--exact".into(),
8814                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8815                "--nocapture".into(),
8816            ],
8817            env: vec![],
8818            reserved: false,
8819            reserved_prefixes: vec![],
8820            protocol: ModuleProtocol::None,
8821            overlap: Default::default(),
8822        }
8823    }
8824
8825    #[tokio::test]
8826    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8827        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8828        let supervisor =
8829            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8830                .with_live_children_record(home.join("live-children.json"));
8831        crate::live_children::fail_next_file_sync();
8832        let result = supervisor.spawn(spec());
8833        let failed = result.is_err();
8834        if let Ok(module) = result {
8835            module.stop().await.unwrap();
8836        }
8837        assert!(failed, "record failure must fail spawn");
8838        let pid = LAST_PID.get().expect("the real OS child was created");
8839        assert!(
8840            subc_os::Process::open(pid).unwrap().is_none(),
8841            "child {pid} survived rejected admission or was not reaped"
8842        );
8843    }
8844
8845    #[cfg(target_os = "macos")]
8846    #[test]
8847    #[ignore = "manual on-device latency measurement"]
8848    fn durable_record_write_latency_50() {
8849        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8850        let path = home.join("live-children.json");
8851        let children = vec![crate::live_children::LiveChild {
8852            module_id: "latency-worker".into(),
8853            pid: std::process::id(),
8854            protocol: ModuleProtocol::None,
8855            start_time: Some(1),
8856            executable: None,
8857            cgroup_name: None,
8858        }];
8859        let mut samples = Vec::new();
8860        for _ in 0..50 {
8861            let start = std::time::Instant::now();
8862            crate::live_children::write_record(&path, &children).unwrap();
8863            samples.push(start.elapsed());
8864        }
8865        samples.sort();
8866        eprintln!(
8867            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8868            (samples[24] + samples[25]) / 2,
8869            samples[47]
8870        );
8871    }
8872
8873    #[cfg(windows)]
8874    #[test]
8875    fn windows_store_touch_fixture() {
8876        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8877            std::fs::write(path, b"module ran").unwrap();
8878        }
8879    }
8880
8881    #[cfg(windows)]
8882    #[tokio::test]
8883    async fn windows_daemon_fixture() {
8884        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8885            return;
8886        };
8887        let mut spec = spec();
8888        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8889        let supervisor =
8890            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8891        let _module = supervisor.spawn(spec).unwrap();
8892    }
8893
8894    #[cfg(windows)]
8895    #[test]
8896    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8897        use std::io::Read;
8898        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8899        // An outer job models a scheduler/service host. The daemon's root and
8900        // root job must nest without requiring breakaway from this job.
8901        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8902        command
8903            .args([
8904                "--exact",
8905                "supervise::spawn_record_tests::windows_daemon_fixture",
8906                "--nocapture",
8907            ])
8908            .env(
8909                "SUBC_TEST_DAEMON_BARRIER",
8910                listener.local_addr().unwrap().to_string(),
8911            );
8912        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8913        listener.set_nonblocking(true).unwrap();
8914        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8915        let mut stream = loop {
8916            if let Ok((stream, _)) = listener.accept() {
8917                break stream;
8918            }
8919            assert!(
8920                daemon.child.try_wait().unwrap().is_none(),
8921                "daemon failed before rendezvous"
8922            );
8923            assert!(
8924                std::time::Instant::now() < deadline,
8925                "daemon did not reach CreateProcess failpoint"
8926            );
8927            std::thread::yield_now();
8928        };
8929        stream
8930            .set_read_timeout(Some(Duration::from_secs(30)))
8931            .unwrap();
8932        let mut pid = [0; 4];
8933        stream.read_exact(&mut pid).unwrap();
8934        let pid = u32::from_ne_bytes(pid);
8935        assert!(subc_jobobject::process_exists(pid));
8936        // Terminate only the daemon, not its outer job: otherwise the host job
8937        // would also kill a child whose daemon never joined its root job.
8938        daemon.child.kill().unwrap();
8939        daemon.child.wait().unwrap();
8940        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8941        if !exited {
8942            daemon.job.terminate().unwrap();
8943        }
8944        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8945    }
8946
8947    #[cfg(windows)]
8948    #[tokio::test]
8949    async fn windows_containment_failure_never_resumes() {
8950        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8951        let sentinel = home.join("store-touched");
8952        let mut spec = spec();
8953        spec.env
8954            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8955        spec.env.push((
8956            "SUBC_TEST_STORE_TOUCH".into(),
8957            sentinel.display().to_string(),
8958        ));
8959        let supervisor =
8960            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8961        let result = supervisor.spawn(spec);
8962        // A version that resumes despite assignment failure must have time to
8963        // execute its sentinel before dropping the returned module kills it.
8964        if let Ok(module) = result {
8965            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8966                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8967                    tokio::task::yield_now().await;
8968                }
8969            })
8970            .await;
8971            let _ = module.stop().await;
8972            panic!(
8973                "assignment failure resumed a module; store touched={}",
8974                sentinel.exists()
8975            );
8976        }
8977        assert!(
8978            !sentinel.exists(),
8979            "refused module executed store-touch sentinel"
8980        );
8981        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8982    }
8983
8984    #[cfg(windows)]
8985    #[tokio::test]
8986    async fn windows_nested_daemon_fixture() {
8987        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8988            return;
8989        };
8990        let mut spec = spec();
8991        spec.env.push((
8992            "SUBC_TEST_STORE_TOUCH".into(),
8993            sentinel.to_str().unwrap().into(),
8994        ));
8995        let supervisor =
8996            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8997        let module = supervisor
8998            .spawn(spec)
8999            .expect("per-module job must nest below daemon and host jobs");
9000        tokio::time::timeout(Duration::from_secs(10), async {
9001            while !PathBuf::from(&sentinel).exists() {
9002                tokio::task::yield_now().await;
9003            }
9004        })
9005        .await
9006        .unwrap();
9007        module.stop().await.unwrap();
9008    }
9009
9010    #[cfg(windows)]
9011    #[test]
9012    fn windows_module_job_nests_under_host_job() {
9013        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9014        let sentinel = home.join("nested-store-touched");
9015        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9016        command
9017            .args([
9018                "--exact",
9019                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9020                "--nocapture",
9021            ])
9022            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9023        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9024        assert!(daemon.child.wait().unwrap().success());
9025        assert!(sentinel.exists(), "nested protocol:none module never ran");
9026    }
9027}
9028
9029#[cfg(target_os = "linux")]
9030pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9031    use subc_cgroup::KillOutcome;
9032    match subc_cgroup::kill_module(placement, module_id) {
9033        KillOutcome::Killed => {}
9034        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9035            debug!(
9036                module_id,
9037                "cgroup tree kill unavailable; using direct-child kill"
9038            );
9039        }
9040        KillOutcome::IoError { path, error } => {
9041            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9042        }
9043    }
9044}
9045
9046/// Assign a freshly spawned Windows child without ever resuming on failure.
9047///
9048/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9049/// child assigned **while it is still suspended** (step 1 is
9050/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9051/// the child's roster admission is durable. Failed containment kills and reaps
9052/// the suspended child instead of running an unprotected module.
9053#[cfg(windows)]
9054fn contain_spawned_child(
9055    child: &mut Child,
9056    spec: &ModuleSpec,
9057) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9058    let assignment = || {
9059        let job = subc_jobobject::JobObject::new()?;
9060        #[cfg(test)]
9061        spawn_record_tests::assignment_failpoint(spec)?;
9062        job.assign(child)?;
9063        Ok::<_, io::Error>(job)
9064    };
9065    match assignment() {
9066        Ok(job) => Ok(Some(job)),
9067        Err(source) => {
9068            kill_and_reap_rejected_child(child);
9069            Err(SuperviseError::Spawn {
9070                program: spec.program.clone(),
9071                source: io::Error::other(format!(
9072                    "Windows module job containment failed: {source}"
9073                )),
9074                cgroup_path: None,
9075            })
9076        }
9077    }
9078}
9079
9080/// Rejection is synchronous: do not return a spawn error while the child can
9081/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9082fn kill_and_reap_rejected_child(child: &mut Child) {
9083    #[cfg(unix)]
9084    if let Some(pid) = child
9085        .id()
9086        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9087    {
9088        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9089    }
9090    let _ = child.start_kill();
9091    while matches!(child.try_wait(), Ok(None)) {
9092        std::thread::yield_now();
9093    }
9094}
9095
9096/// Resume a suspended child, killing it if it cannot be started.
9097///
9098/// A suspended process holds a pid and does nothing, so there is no useful
9099/// state to return: the caller gets an error and the spawn fails.
9100#[cfg(windows)]
9101fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9102    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9103        // Kill it here rather than leaving a suspended process for the caller
9104        // to notice; `kill_on_drop` would eventually do this, but the module
9105        // would have been reported as running in between.
9106        let _ = std::process::Command::new("taskkill.exe")
9107            .args(["/PID", &pid.to_string(), "/T", "/F"])
9108            .stdin(Stdio::null())
9109            .stdout(Stdio::null())
9110            .stderr(Stdio::null())
9111            .status();
9112        return Err(SuperviseError::Spawn {
9113            program: spec.program.clone(),
9114            source,
9115            cgroup_path: None,
9116        });
9117    }
9118    Ok(())
9119}
9120
9121#[cfg(target_os = "linux")]
9122fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9123    match placement.remove_module(module_id) {
9124        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9125        Err(error) => warn!(
9126            module_id,
9127            error = %error,
9128            "could not remove module cgroup after process exit; continuing teardown"
9129        ),
9130    }
9131}
9132
9133#[cfg(target_os = "linux")]
9134async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9135    // Reaping the direct child is not proof its descendants exited. End the
9136    // residual tree and wait for the kernel's population fact before rmdir;
9137    // otherwise a successful parent wait leaks a directory on each restart.
9138    if matches!(
9139        subc_cgroup::kill_module(Some(placement), module_id),
9140        subc_cgroup::KillOutcome::Killed
9141    ) {
9142        if let Ok(path) = placement.module_path(module_id) {
9143            while std::fs::read_to_string(path.join("cgroup.events"))
9144                .ok()
9145                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9146            {
9147                sleep(Duration::from_millis(1)).await;
9148            }
9149        }
9150    }
9151    remove_module_cgroup(placement, module_id);
9152}
9153
9154#[cfg(target_os = "linux")]
9155fn apply_cgroup_placement(
9156    command: &mut Command,
9157    spec: &ModuleSpec,
9158    path: &std::path::Path,
9159) -> Result<(), SuperviseError> {
9160    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9161        module_id: spec.module_id.clone(),
9162        source,
9163    })
9164}
9165
9166fn capture_retention(spec: &ModuleSpec) -> Retention {
9167    let defaults = Retention::default();
9168    let value = |name: &str| {
9169        spec.env
9170            .iter()
9171            .rev()
9172            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9173    };
9174    Retention {
9175        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9176            .and_then(|value| value.parse().ok())
9177            .unwrap_or(defaults.max_file_mb),
9178        keep: value(CAPTURE_KEEP_ENV)
9179            .and_then(|value| value.parse().ok())
9180            .unwrap_or(defaults.keep),
9181        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9182            .and_then(|value| value.parse().ok())
9183            .unwrap_or(defaults.max_age_days),
9184    }
9185}
9186
9187/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9188/// module's registration to the exact process the supervisor spawned.
9189fn generate_launch_nonce() -> Result<String, SuperviseError> {
9190    let mut bytes = [0u8; 32];
9191    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9192        reason: source.to_string(),
9193    })?;
9194    let mut hex = String::with_capacity(64);
9195    for b in bytes {
9196        use std::fmt::Write;
9197        let _ = write!(hex, "{b:02x}");
9198    }
9199    Ok(hex)
9200}
9201
9202/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9203/// signal about how many leading bytes matched.
9204fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9205    if a.len() != b.len() {
9206        return false;
9207    }
9208    let mut diff = 0u8;
9209    for (x, y) in a.iter().zip(b.iter()) {
9210        diff |= x ^ y;
9211    }
9212    diff == 0
9213}
9214
9215/// The kernel's image after an acknowledged exec, shared by ordinary launches
9216/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9217/// not identities inferred from a configured pathname.
9218#[cfg(not(windows))]
9219fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9220    let process = subc_os::Process::open(pid).ok().flatten()?;
9221    #[cfg(target_os = "linux")]
9222    return sample_after_linux_exec(
9223        || {
9224            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9225            let committed = linux_exec_committed(&stat);
9226            #[cfg(test)]
9227            spawn_image_tests::report_exec_state(committed);
9228            committed
9229        },
9230        || process.observe(),
9231    );
9232    #[cfg(not(target_os = "linux"))]
9233    process.observe()
9234}
9235
9236/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9237/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9238/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9239#[cfg(target_os = "linux")]
9240fn linux_exec_committed(stat: &str) -> Option<bool> {
9241    const PF_FORKNOEXEC: u64 = 0x40;
9242    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9243    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9244}
9245
9246#[cfg(target_os = "linux")]
9247fn sample_after_linux_exec(
9248    mut exec_committed: impl FnMut() -> Option<bool>,
9249    observe: impl FnOnce() -> Option<subc_os::Observation>,
9250) -> Option<subc_os::Observation> {
9251    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9252    while !exec_committed()? {
9253        // A failed or indefinitely stalled exec must not publish the inherited
9254        // image as authority to signal a future orphan. Missing identity keeps
9255        // the existing conservative orphan-sweep behavior.
9256        if std::time::Instant::now() >= deadline {
9257            return None;
9258        }
9259        std::thread::sleep(Duration::from_micros(50));
9260    }
9261    observe()
9262}
9263
9264#[cfg(all(test, target_os = "linux"))]
9265mod spawn_image_tests {
9266    use super::*;
9267    use std::cell::{Cell, RefCell};
9268
9269    enum ExecEvent {
9270        Uncommitted,
9271        Image(Option<subc_os::Observation>),
9272    }
9273
9274    thread_local! {
9275        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9276    }
9277
9278    pub(super) fn report_exec_state(committed: Option<bool>) {
9279        if committed == Some(false) {
9280            EXEC_EVENTS.with(|events| {
9281                if let Some(events) = events.borrow_mut().take() {
9282                    let _ = events.send(ExecEvent::Uncommitted);
9283                }
9284            });
9285        }
9286    }
9287
9288    #[test]
9289    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9290        use std::io::{Read, Write};
9291        use std::os::fd::AsRawFd;
9292
9293        let descriptor = std::fs::File::open("/dev/null").unwrap();
9294        let mut command = std::process::Command::new("/bin/sleep");
9295        command.arg("30");
9296        let (mut ready, mut release) =
9297            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9298                .unwrap();
9299        let spawn = std::thread::spawn(move || command.spawn());
9300        let mut message = [0u8; 8];
9301        ready.read_exact(&mut message).unwrap();
9302        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9303        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9304        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9305        let (events, received) = std::sync::mpsc::channel();
9306        let observer = std::thread::spawn(move || {
9307            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9308            let image = observe_spawned_image(pid);
9309            let _ = events.send(ExecEvent::Image(image));
9310        });
9311        // Release only once the production observer has seen the uncommitted
9312        // exec, or has incorrectly returned the inherited image. Either way,
9313        // release and reap before asserting so a regression cannot leak a child.
9314        let first = received.recv_timeout(Duration::from_secs(5));
9315        release.write_all(&[1]).unwrap();
9316        let mut child = spawn.join().unwrap().unwrap();
9317        let after = match first {
9318            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9319                Ok(ExecEvent::Image(image)) => image,
9320                _ => None,
9321            },
9322            Ok(ExecEvent::Image(image)) => image,
9323            Err(_) => None,
9324        };
9325        observer.join().unwrap();
9326        child.kill().unwrap();
9327        child.wait().unwrap();
9328
9329        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9330        assert_eq!(
9331            inherited.unwrap().executable,
9332            subc_os::file_identity(&std::env::current_exe().unwrap())
9333        );
9334        assert_eq!(
9335            after.unwrap().executable,
9336            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9337        );
9338    }
9339
9340    #[test]
9341    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9342        let committed = Cell::new(false);
9343        let inherited = subc_os::Observation {
9344            start_time: 23,
9345            executable: Some(subc_os::FileIdentity {
9346                device: 29,
9347                inode: 31,
9348            }),
9349        };
9350        let module = subc_os::Observation {
9351            start_time: 23,
9352            executable: Some(subc_os::FileIdentity {
9353                device: 29,
9354                inode: 37,
9355            }),
9356        };
9357        let polls = Cell::new(0);
9358        let image = sample_after_linux_exec(
9359            || {
9360                polls.set(polls.get() + 1);
9361                // Hold the inherited image through the first poll, as in the
9362                // posix_spawn parent-wake window, then commit the module image.
9363                committed.set(polls.get() == 2);
9364                Some(committed.get())
9365            },
9366            || Some(if committed.get() { module } else { inherited }),
9367        );
9368        assert_eq!(image, Some(module));
9369        assert_eq!(polls.get(), 2);
9370    }
9371
9372    #[test]
9373    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9374        assert_eq!(
9375            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9376            Some(false)
9377        );
9378        assert_eq!(
9379            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9380            Some(true)
9381        );
9382        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9383        assert_eq!(linux_exec_committed("17 (a) R"), None);
9384    }
9385
9386    #[test]
9387    fn unreadable_exec_state_never_samples_an_inherited_image() {
9388        assert_eq!(
9389            sample_after_linux_exec(
9390                || None,
9391                || panic!("must not sample before exec confirmation")
9392            ),
9393            None
9394        );
9395    }
9396
9397    #[test]
9398    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9399        assert_eq!(
9400            sample_after_linux_exec(
9401                || Some(false),
9402                || panic!("must not sample after an exec timeout")
9403            ),
9404            None
9405        );
9406    }
9407}
9408
9409/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9410/// different executable after the no-delete capture was taken.
9411#[cfg(windows)]
9412fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9413    let mut program = spec.program.clone();
9414    if program.is_absolute() || program.components().count() > 1 {
9415        // An explicit path can name a PE image with no extension. Try that
9416        // exact file first, as Command does, before falling back to .exe.
9417        match subc_os::ExecutableCapture::open(&program) {
9418            Ok(capture) => return Ok(capture),
9419            Err(error)
9420                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9421            Err(error) => return Err(error),
9422        }
9423        program.set_extension("exe");
9424        return subc_os::ExecutableCapture::open(&program);
9425    }
9426    if program.extension().is_none() {
9427        program.set_extension("exe");
9428    }
9429    let custom_path = spec
9430        .env
9431        .iter()
9432        .rev()
9433        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9434    let mut directories = Vec::new();
9435    // These are Windows' default executable search locations. A configured
9436    // PATH still overrides the inherited PATH, not the system directories.
9437    if let Some(parent) = std::env::current_exe()?.parent() {
9438        directories.push(parent.to_path_buf());
9439    }
9440    directories.push(std::env::current_dir()?);
9441    if let Some(windows) = std::env::var_os("SystemRoot") {
9442        let windows = PathBuf::from(windows);
9443        directories.push(windows.join("System32"));
9444        directories.push(windows.clone());
9445    }
9446    let path = custom_path
9447        .map(|(_, value)| std::ffi::OsString::from(value))
9448        .or_else(|| std::env::var_os("PATH"));
9449    if let Some(path) = path {
9450        directories.extend(std::env::split_paths(&path));
9451    }
9452    for directory in directories {
9453        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9454            Ok(capture) => return Ok(capture),
9455            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9456            Err(error) => return Err(error),
9457        }
9458    }
9459    Err(io::Error::new(
9460        io::ErrorKind::NotFound,
9461        "resolved executable not found",
9462    ))
9463}
9464
9465fn spawn_and_mark_running(
9466    spec: &ModuleSpec,
9467    runtime: &SupervisorRuntimeConfig,
9468    snapshot: &SharedSnapshot,
9469) -> Result<SupervisedChild, SuperviseError> {
9470    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9471    let child = spawn_child(
9472        spec,
9473        runtime.connection_file_path.as_deref(),
9474        runtime.supervisor_handle.as_ref(),
9475        &runtime.stderr_ring,
9476        runtime.capture_logs_dir.as_deref(),
9477        &runtime.child_roster,
9478        #[cfg(target_os = "linux")]
9479        runtime.cgroup_placement.as_ref(),
9480    )?;
9481    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9482    if let Some(spawn) = activity_spawn {
9483        spawn.complete(snapshot);
9484    }
9485    Ok(child)
9486}
9487
9488enum RegistrationWaitOutcome {
9489    Registered,
9490    Exited(ExitReport),
9491    TimedOut,
9492}
9493
9494struct ReloadRegistrationFailure {
9495    exit_report: ExitReport,
9496    reason: String,
9497}
9498
9499#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9500enum BusyGaugeObservation {
9501    Quiescent,
9502    Busy,
9503    Omitted,
9504}
9505
9506/// What one health answer said about one declared busy gauge.
9507#[derive(Debug, Clone, PartialEq, Eq)]
9508enum GaugeVerdict {
9509    /// The gauge read 0: this gauge is not what holds the drain.
9510    Zero,
9511    /// The gauge read a positive count.
9512    NonZero(u64),
9513    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9514    /// a negative number). It counts as busy, because the daemon cannot tell
9515    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9516    NotACount(String),
9517    /// The health answer had no metrics object, or one without this gauge.
9518    Missing,
9519    /// The probe produced no health answer to read; carries the probe error's
9520    /// evidence label (`no-answer` for a probe that timed out).
9521    ProbeFailed(&'static str),
9522}
9523
9524impl GaugeVerdict {
9525    fn reads_quiet(&self) -> bool {
9526        matches!(self, Self::Zero)
9527    }
9528}
9529
9530impl fmt::Display for GaugeVerdict {
9531    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9532        match self {
9533            Self::Zero => f.write_str("zero"),
9534            Self::NonZero(value) => write!(f, "non_zero({value})"),
9535            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9536            Self::Missing => f.write_str("missing_from_health_body"),
9537            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9538        }
9539    }
9540}
9541
9542/// The longest JSON text kept for a gauge value that is not a count, so a
9543/// module reporting a large object as a gauge cannot flood the drain log.
9544const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9545
9546/// Read each declared gauge out of one health answer's metrics.
9547fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9548    let metrics = metrics.and_then(Value::as_object);
9549    gauges
9550        .iter()
9551        .map(
9552            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9553                None => GaugeVerdict::Missing,
9554                Some(value) => match value.as_u64() {
9555                    Some(0) => GaugeVerdict::Zero,
9556                    Some(count) => GaugeVerdict::NonZero(count),
9557                    None => {
9558                        let mut text = value.to_string();
9559                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9560                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9561                            while !text.is_char_boundary(end) {
9562                                end -= 1;
9563                            }
9564                            text.truncate(end);
9565                            text.push_str("...");
9566                        }
9567                        GaugeVerdict::NotACount(text)
9568                    }
9569                },
9570            },
9571        )
9572        .collect()
9573}
9574
9575/// The drain's overall reading of one health answer: omitted when any
9576/// declared gauge is missing, quiescent only when every gauge reads 0.
9577fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9578    if verdicts
9579        .iter()
9580        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9581    {
9582        BusyGaugeObservation::Omitted
9583    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9584        BusyGaugeObservation::Quiescent
9585    } else {
9586        BusyGaugeObservation::Busy
9587    }
9588}
9589
9590/// One declared gauge's history over a drain: the latest verdict on it, and
9591/// the latest value actually read for it (which a later failed probe or
9592/// missing gauge does not erase). Times are offsets from the start of the
9593/// drain's wait.
9594#[derive(Debug, Clone, Default, PartialEq, Eq)]
9595struct GaugeTrace {
9596    latest: Option<(GaugeVerdict, Duration)>,
9597    last_value: Option<(GaugeVerdict, Duration)>,
9598}
9599
9600/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9601/// its ceiling can say in its timeout line which declared gauge held it and
9602/// why. Without it, a gauge that read busy, a gauge the module never
9603/// reported, and a probe that never got an answer all looked the same: a
9604/// drain that ended at its ceiling with nothing in flight.
9605#[derive(Debug, Clone, Default, PartialEq, Eq)]
9606struct DrainGaugeTrace {
9607    /// Health probes the drain sent.
9608    probes: u32,
9609    /// Each declared gauge, in declaration order.
9610    gauges: Vec<(String, GaugeTrace)>,
9611}
9612
9613impl DrainGaugeTrace {
9614    fn new(gauges: &[String]) -> Self {
9615        Self {
9616            probes: 0,
9617            gauges: gauges
9618                .iter()
9619                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9620                .collect(),
9621        }
9622    }
9623
9624    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9625    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9626        self.probes = self.probes.saturating_add(1);
9627        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9628            trace.latest = Some((verdict.clone(), at));
9629            if matches!(
9630                verdict,
9631                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9632            ) {
9633                trace.last_value = Some((verdict.clone(), at));
9634            }
9635        }
9636    }
9637}
9638
9639/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
9640/// declared gauge, separated by ` | `. A value is shown as its verdict
9641/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
9642/// and whether that counted as busy.
9643impl fmt::Display for DrainGaugeTrace {
9644    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9645        if self.gauges.is_empty() {
9646            return f.write_str("none declared");
9647        }
9648        for (index, (name, trace)) in self.gauges.iter().enumerate() {
9649            if index > 0 {
9650                f.write_str(" | ")?;
9651            }
9652            write!(f, "{name}: ")?;
9653            match &trace.latest {
9654                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9655                None => f.write_str("never probed")?,
9656            }
9657            match &trace.last_value {
9658                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9659                None => f.write_str("; no value ever read")?,
9660            }
9661        }
9662        Ok(())
9663    }
9664}
9665
9666pub(crate) fn declared_busy_gauges(
9667    registry: &Registry,
9668    module_id: &str,
9669) -> Result<Vec<String>, SuperviseError> {
9670    busy_gauges_of(
9671        registry
9672            .get_module(module_id)
9673            .map_err(SuperviseError::Registry)?,
9674    )
9675}
9676
9677/// [`declared_busy_gauges`] for the registration a connection holds, in any
9678/// slot: after cutover the incumbent is no longer the id's active
9679/// registration, and its own manifest is the one that names its gauges.
9680fn declared_busy_gauges_for_connection(
9681    registry: &Registry,
9682    connection_id: ConnectionId,
9683) -> Result<Vec<String>, SuperviseError> {
9684    busy_gauges_of(
9685        registry
9686            .get_module_by_connection(connection_id)
9687            .map_err(SuperviseError::Registry)?,
9688    )
9689}
9690
9691fn busy_gauges_of(
9692    registration: Option<crate::registry::ModuleRegistration>,
9693) -> Result<Vec<String>, SuperviseError> {
9694    let Some(registration) = registration else {
9695        return Ok(Vec::new());
9696    };
9697    let Some(self_signals) = registration.manifest.self_signals else {
9698        return Ok(Vec::new());
9699    };
9700
9701    let mut gauges = Vec::new();
9702    for declaration in self_signals {
9703        if declaration.kind != SelfSignalKind::Busy {
9704            continue;
9705        }
9706        match declaration.anchored_to {
9707            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9708                gauges.extend(declared)
9709            }
9710            _ => {
9711                // An invalid Busy anchor is fail-safe: the empty name cannot be
9712                // present in a conforming health report, so this drain stays busy.
9713                gauges.push(String::new());
9714            }
9715        }
9716    }
9717    Ok(gauges)
9718}
9719
9720/// Wait for `endpoint` to have nothing in flight and, when the module declares
9721/// busy gauges, for a health probe to report them quiet. The probe is addressed
9722/// by `scope`: a swap's superseded incumbent must be asked about its own
9723/// gauges, and by module id the probe would reach the promoted candidate.
9724/// While the gauges read busy, the probe repeats every
9725/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9726/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
9727/// timeout line.
9728#[allow(clippy::too_many_arguments)]
9729async fn wait_for_forwarding_quiescence(
9730    forwarding: &ForwardingTable,
9731    module_id: &str,
9732    runtime: &SupervisorRuntimeConfig,
9733    endpoint: crate::ModuleEndpointId,
9734    deadline: Instant,
9735    busy_gauges: &[String],
9736    scope: DrainScope,
9737    trace: &mut DrainGaugeTrace,
9738) -> Result<bool, SuperviseError> {
9739    let wait_started = Instant::now();
9740    let mut gauges_quiescent = busy_gauges.is_empty();
9741    let mut next_probe_at = wait_started;
9742    let mut omission_reported = false;
9743
9744    loop {
9745        let now = Instant::now();
9746        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9747            let report = match scope {
9748                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9749                DrainScope::Endpoint(endpoint) => {
9750                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9751                }
9752            };
9753            let verdicts = match &report {
9754                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9755                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9756            };
9757            trace.record(&verdicts, wait_started.elapsed());
9758            gauges_quiescent = match report {
9759                Ok(_) => match busy_gauge_observation(&verdicts) {
9760                    BusyGaugeObservation::Quiescent => true,
9761                    BusyGaugeObservation::Busy => false,
9762                    BusyGaugeObservation::Omitted => {
9763                        // Once per drain: a module that never reports a gauge
9764                        // it declared would otherwise say so on every probe.
9765                        if !omission_reported {
9766                            forwarding
9767                                .counters()
9768                                .increment_drains_with_undeclared_gauge();
9769                            let missing = busy_gauges
9770                                .iter()
9771                                .zip(&verdicts)
9772                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9773                                .map(|(gauge, _)| gauge.as_str())
9774                                .collect::<Vec<_>>()
9775                                .join(",");
9776                            warn!(
9777                                module_id,
9778                                missing_gauges = %missing,
9779                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9780                            );
9781                            omission_reported = true;
9782                        }
9783                        false
9784                    }
9785                },
9786                Err(err) => {
9787                    warn!(
9788                        module_id,
9789                        error = %err,
9790                        "drain health.check did not produce declared busy gauges; treating module as busy"
9791                    );
9792                    false
9793                }
9794            };
9795            // Counted from the probe's completion: the probe above was awaited,
9796            // so no second probe can start while one is still outstanding.
9797            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9798        }
9799
9800        let in_flight = forwarding
9801            .endpoint_in_flight_count(endpoint)
9802            .map_err(SuperviseError::Forwarding)?;
9803        if in_flight == 0 && gauges_quiescent {
9804            return Ok(true);
9805        }
9806
9807        let now = Instant::now();
9808        if now >= deadline {
9809            return Ok(false);
9810        }
9811        let mut wait = deadline
9812            .saturating_duration_since(now)
9813            .min(REGISTRY_RELEASE_POLL);
9814        if !busy_gauges.is_empty() {
9815            wait = wait.min(next_probe_at.saturating_duration_since(now));
9816        }
9817        sleep(wait).await;
9818    }
9819}
9820
9821/// The wait between drain busy-gauge probes: the larger of
9822/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
9823/// drain's own in-flight poll interval.
9824fn drain_busy_reprobe_interval() -> Duration {
9825    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9826}
9827
9828/// Logs why a drain gave up: the drain waited its full timeout without the
9829/// module going quiet, and teardown is being forced.
9830///
9831/// The line names two kinds of evidence, because a bare "timed out" hides
9832/// causes that need different fixes:
9833/// - `holdouts`: the requests and routes still open when the timeout hit,
9834///   and on which connections. One stuck call is a module bug; a held stream
9835///   on every session is a module that should end its streams when it
9836///   receives `module.draining`.
9837/// - `gauges`: for each busy gauge the module declares, its latest reading
9838///   and how many probes ran. This separates a module that kept reporting
9839///   itself busy, one whose health answer never contained the gauge, and one
9840///   that never answered the probes, which otherwise all look like a timeout
9841///   with nothing in flight.
9842fn warn_drain_timed_out(
9843    module_id: &str,
9844    drain_timeout: Duration,
9845    reason: RouteCloseReason,
9846    holdouts: &crate::forwarding::DrainHoldouts,
9847    gauges: &DrainGaugeTrace,
9848) {
9849    warn!(
9850        module_id = %module_id,
9851        waited = ?drain_timeout,
9852        ?reason,
9853        held_requests = holdouts.requests,
9854        held_routes = holdouts.routes,
9855        total_routes = holdouts.total_routes,
9856        top_connections = ?holdouts.top_connections,
9857        // `module_channel:corr`, so the module can find each held request
9858        // in its own log; capped, so `held_requests` is the full count.
9859        held = %holdouts
9860            .held
9861            .iter()
9862            .map(|(channel, corr)| format!("{channel}:{corr}"))
9863            .collect::<Vec<_>>()
9864            .join(","),
9865        gauge_probes = gauges.probes,
9866        busy_gauges = %gauges,
9867        "route drain timed out before request quiescence; forcing teardown"
9868    );
9869}
9870
9871/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9872///
9873/// `Ok` is always honest and passed straight through -- the wait actually measured
9874/// in-flight state. `Err` means the wait produced no measurement at all (the
9875/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9876/// constant: the drain did not complete. Never recomputed from route state, never a
9877/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9878fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9879    match wait_result {
9880        Ok(drained) => *drained,
9881        Err(_) => false,
9882    }
9883}
9884
9885fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9886    for released in released_routes {
9887        let frame = match Frame::build_with_version(
9888            released.negotiated_ver,
9889            FrameType::Goodbye,
9890            control_flags(),
9891            released.channel,
9892            released.epoch,
9893            0,
9894            Vec::new(),
9895        ) {
9896            Ok(frame) => frame,
9897            Err(err) => {
9898                warn!(
9899                    route_channel = released.channel,
9900                    error = %err,
9901                    "failed to build supervisor drain route GOODBYE frame"
9902                );
9903                continue;
9904            }
9905        };
9906        if !released.close_on_delivery_failure() {
9907            crate::forwarding::send_module_route_goodbye(
9908                &forwarding.counters(),
9909                &released.sink,
9910                frame,
9911                released.module_id.as_deref(),
9912                "supervisor drain",
9913            );
9914            continue;
9915        }
9916        if let Err(err) = released.sink.try_send(frame) {
9917            warn!(
9918                target_connection_id = released.connection_id.get(),
9919                route_channel = released.channel,
9920                error = %err,
9921                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9922            );
9923            let _ = forwarding.escalate_client_delivery_failure(
9924                released.connection_id,
9925                released.channel,
9926                released.epoch,
9927                CloseReason::new(
9928                    "route_goodbye_delivery_failed",
9929                    format!(
9930                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9931                        released.channel
9932                    ),
9933                ),
9934                crate::forwarding::UndeliveredFrame {
9935                    module_id: released.module_id.as_deref(),
9936                    sink: &released.sink,
9937                },
9938            );
9939        }
9940    }
9941}
9942
9943fn send_module_draining(
9944    module_id: &str,
9945    reason: RouteCloseReason,
9946    deadline_ms: u64,
9947    target: &ModuleDrainTarget,
9948) {
9949    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9950        reason,
9951        deadline_ms,
9952    }) {
9953        Ok(body) => body,
9954        Err(err) => {
9955            warn!(
9956                module_id,
9957                error = %err,
9958                "failed to encode module draining command"
9959            );
9960            return;
9961        }
9962    };
9963    let frame = match Frame::build_with_version(
9964        target.negotiated_ver,
9965        FrameType::Push,
9966        control_flags(),
9967        0,
9968        0,
9969        0,
9970        body,
9971    ) {
9972        Ok(frame) => frame,
9973        Err(err) => {
9974            warn!(
9975                module_id,
9976                error = %err,
9977                "failed to build module draining command frame"
9978            );
9979            return;
9980        }
9981    };
9982    if let Err(err) = target.sink.try_send(frame) {
9983        warn!(
9984            module_id,
9985            target_connection_id = target.endpoint.connection_id.get(),
9986            error = %err,
9987            "module draining command was not delivered to peer"
9988        );
9989    }
9990}
9991
9992/// The channel-0 GOODBYE that tells a module its stop is planned.
9993fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9994    match Frame::build_with_version(
9995        negotiated_ver,
9996        FrameType::Goodbye,
9997        control_flags(),
9998        0,
9999        0,
10000        0,
10001        Vec::new(),
10002    ) {
10003        Ok(frame) => Some(frame),
10004        Err(err) => {
10005            warn!(
10006                module_id,
10007                error = %err,
10008                "failed to build module GOODBYE frame"
10009            );
10010            None
10011        }
10012    }
10013}
10014
10015/// Send every registered module connection its module GOODBYE at daemon
10016/// shutdown, then request that connection's close.
10017///
10018/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10019/// before EOF, so the GOODBYE must reach the socket before the close. A close
10020/// request does not wait for the connection's queued frames: its writer gets a
10021/// bounded grace after the close, is aborted if it overruns it, and the daemon
10022/// process may exit before that grace ends. So with `wait_for_flush`, each
10023/// connection is closed only after its writer has acknowledged writing the
10024/// GOODBYE, or once a short shared budget runs out, so one module that is not
10025/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10026/// are only queued, for a shutdown the operator has told to stop waiting.
10027/// A connection that is already gone is skipped.
10028#[cfg(unix)]
10029async fn send_module_goodbyes_for_daemon_shutdown(
10030    forwarding: &Arc<ForwardingTable>,
10031    reason: &CloseReason,
10032    wait_for_flush: bool,
10033) {
10034    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10035    let targets = match forwarding.module_connections() {
10036        Ok(targets) => targets,
10037        Err(err) => {
10038            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10039            return;
10040        }
10041    };
10042    let deadline = Instant::now() + GOODBYE_BUDGET;
10043    let mut sends = tokio::task::JoinSet::new();
10044    for target in targets {
10045        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10046            continue;
10047        };
10048        if !wait_for_flush {
10049            if let Err(err) = target.sink.try_send(frame) {
10050                debug!(
10051                    module_id = %target.module_id,
10052                    error = %err,
10053                    "shutdown module GOODBYE was not queued"
10054                );
10055            }
10056            continue;
10057        }
10058        let forwarding = Arc::clone(forwarding);
10059        let reason = reason.clone();
10060        sends.spawn(async move {
10061            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10062                Ok(Ok(())) => {}
10063                Ok(Err(err)) => debug!(
10064                    module_id = %target.module_id,
10065                    error = %err,
10066                    "module connection closed before its shutdown GOODBYE was written"
10067                ),
10068                Err(_) => warn!(
10069                    module_id = %target.module_id,
10070                    budget = ?GOODBYE_BUDGET,
10071                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10072                ),
10073            }
10074            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10075        });
10076    }
10077    // Every task ends by the shared deadline, so this wait is bounded too.
10078    while sends.join_next().await.is_some() {}
10079}
10080
10081fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10082    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10083        return;
10084    };
10085    if let Err(err) = target.sink.try_send(frame) {
10086        warn!(
10087            module_id,
10088            target_connection_id = target.endpoint.connection_id.get(),
10089            error = %err,
10090            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10091        );
10092        forwarding.request_connection_close(
10093            target.endpoint.connection_id,
10094            CloseReason::new(
10095                "module_goodbye_delivery_failed",
10096                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10097            ),
10098        );
10099    }
10100}
10101
10102#[derive(Clone, Copy)]
10103struct ForwardingDrainContext<'a> {
10104    spec: &'a ModuleSpec,
10105    runtime: &'a SupervisorRuntimeConfig,
10106    registry: &'a Registry,
10107    scope: DrainScope,
10108}
10109
10110/// Which process a forwarding drain addresses.
10111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10112enum DrainScope {
10113    /// Whatever endpoint is active for the module id: every plain stop,
10114    /// restart and reload. Also moves the module's state to `Draining`.
10115    Active,
10116    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10117    /// module id would resolve to the promoted candidate and leave neither
10118    /// process routable. The module's state is left alone, since the promoted
10119    /// candidate is what it describes and that process is running.
10120    Endpoint(crate::ModuleEndpointId),
10121}
10122
10123/// Whether a child being drained has already been asked to stop by the time
10124/// its drain wait starts.
10125///
10126/// The drain wait is the same budget whatever this says. What it decides is
10127/// whether the supervisor must ask by signal before that wait begins: a child
10128/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10129/// healthy or not.
10130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10131enum StopNotice {
10132    /// The module was sent `module.draining` and a module GOODBYE over its own
10133    /// registered connection, and stops itself.
10134    SentOverConnection,
10135    /// The forwarding drain found no registered connection for the module: a
10136    /// subc child spawned moments ago that has not sent HELLO yet, or a
10137    /// `protocol: "none"` child, which never registers.
10138    NoConnection,
10139    /// This path sends nothing over the module's connection: the supervisor has
10140    /// no forwarding table, or the caller stops the child without a forwarding
10141    /// drain.
10142    NotSent,
10143}
10144
10145async fn begin_forwarding_drain(
10146    spec: &ModuleSpec,
10147    runtime: &SupervisorRuntimeConfig,
10148    registry: &Registry,
10149    snapshot: &SharedSnapshot,
10150    enabled: Option<bool>,
10151    reason: RouteCloseReason,
10152) -> Result<StopNotice, SuperviseError> {
10153    let Some(forwarding) = runtime.forwarding.as_ref() else {
10154        return Err(SuperviseError::ReloadUnavailable {
10155            module_id: spec.module_id.clone(),
10156            reason: "supervisor was not configured with a forwarding table".to_string(),
10157        });
10158    };
10159
10160    begin_forwarding_drain_with(
10161        forwarding,
10162        ForwardingDrainContext {
10163            spec,
10164            runtime,
10165            registry,
10166            scope: DrainScope::Active,
10167        },
10168        snapshot,
10169        enabled,
10170        reason,
10171        runtime.drain_timeout,
10172    )
10173    .await
10174}
10175
10176async fn begin_forwarding_drain_if_configured(
10177    spec: &ModuleSpec,
10178    runtime: &SupervisorRuntimeConfig,
10179    registry: &Registry,
10180    snapshot: &SharedSnapshot,
10181    enabled: Option<bool>,
10182    reason: RouteCloseReason,
10183) -> Result<StopNotice, SuperviseError> {
10184    begin_forwarding_drain_with_timeout(
10185        spec,
10186        runtime,
10187        registry,
10188        snapshot,
10189        enabled,
10190        reason,
10191        runtime.drain_timeout,
10192    )
10193    .await
10194}
10195
10196/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10197/// budget, for paths where the operator overrides the module's configured one
10198/// (`supervisor.restart{drain_timeout_ms}`).
10199async fn begin_forwarding_drain_with_timeout(
10200    spec: &ModuleSpec,
10201    runtime: &SupervisorRuntimeConfig,
10202    registry: &Registry,
10203    snapshot: &SharedSnapshot,
10204    enabled: Option<bool>,
10205    reason: RouteCloseReason,
10206    drain_timeout: Duration,
10207) -> Result<StopNotice, SuperviseError> {
10208    let Some(forwarding) = runtime.forwarding.as_ref() else {
10209        return Ok(StopNotice::NotSent);
10210    };
10211
10212    begin_forwarding_drain_with(
10213        forwarding,
10214        ForwardingDrainContext {
10215            spec,
10216            runtime,
10217            registry,
10218            scope: DrainScope::Active,
10219        },
10220        snapshot,
10221        enabled,
10222        reason,
10223        drain_timeout,
10224    )
10225    .await
10226}
10227
10228async fn begin_forwarding_drain_with(
10229    forwarding: &ForwardingTable,
10230    context: ForwardingDrainContext<'_>,
10231    snapshot: &SharedSnapshot,
10232    enabled: Option<bool>,
10233    reason: RouteCloseReason,
10234    drain_timeout: Duration,
10235) -> Result<StopNotice, SuperviseError> {
10236    let ForwardingDrainContext {
10237        spec,
10238        runtime,
10239        registry,
10240        scope,
10241    } = context;
10242    debug_assert_ne!(reason, RouteCloseReason::Crash);
10243    let terminal = matches!(reason, RouteCloseReason::Disable);
10244    let drain_started_at = Instant::now();
10245    let drain_deadline = drain_started_at + drain_timeout;
10246    let deadline_ms =
10247        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10248    let busy_gauges = match scope {
10249        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10250        DrainScope::Endpoint(endpoint) => {
10251            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10252        }
10253    };
10254
10255    // Admission gate first: route.open/commit and route REQUEST admission are closed
10256    // before the first quiescence check, so the outstanding count can only fall.
10257    let gate_started = Instant::now();
10258    let drain_target = match scope {
10259        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10260        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10261    }
10262    .map_err(SuperviseError::Forwarding)?;
10263    // The instant admission closed, and how long taking the forwarding write
10264    // lock to close it took. The timeout line reports only the quiescence
10265    // wait, so without this a drain that started late looked like one that
10266    // started on time.
10267    info!(
10268        module_id = %spec.module_id,
10269        ?reason,
10270        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10271        connected = drain_target.is_some(),
10272        "module drain began; route admission closed"
10273    );
10274    if scope == DrainScope::Active {
10275        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10276            state.state = ModuleState::Draining;
10277            state.draining_to_replace =
10278                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10279            if let Some(enabled) = enabled {
10280                state.enabled = enabled;
10281            }
10282        })?;
10283    }
10284
10285    let Some(target) = drain_target.as_ref() else {
10286        // Nothing was sent: the module has no registered connection to carry
10287        // `module.draining` or a GOODBYE. The caller must not assume the child
10288        // was asked to stop.
10289        return Ok(StopNotice::NoConnection);
10290    };
10291    {
10292        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10293        #[cfg(feature = "test-support")]
10294        forwarding.wait_before_route_closing_for_test().await;
10295        let routes = forwarding
10296            .endpoint_routes(target.endpoint)
10297            .map_err(SuperviseError::Forwarding)?;
10298        let routes_notified = routes.len();
10299        crate::control::send_route_control_pushes(
10300            forwarding,
10301            routes.clone(),
10302            ClientControlPush::RouteClosing {
10303                module_id: spec.module_id.clone(),
10304                channels: Vec::new(),
10305                reason,
10306            },
10307        );
10308        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10309
10310        // `route.closing` was just sent above: from here on every return path,
10311        // including an early one, MUST send `route.closed` before propagating
10312        // anything else. A client holds `closing` as a promise that a verdict is
10313        // coming; leaving early without `closed` strands it waiting forever, since
10314        // `closing` carries no timeout of its own.
10315        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10316        let wait_result = wait_for_forwarding_quiescence(
10317            forwarding,
10318            &spec.module_id,
10319            runtime,
10320            target.endpoint,
10321            drain_deadline,
10322            &busy_gauges,
10323            scope,
10324            &mut gauge_trace,
10325        )
10326        .await;
10327        let drained = drained_after_quiescence_wait(&wait_result);
10328        if let Err(err) = &wait_result {
10329            error!(
10330                module_id = %spec.module_id,
10331                ?reason,
10332                error = %err,
10333                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10334            );
10335        } else if !drained {
10336            // Read before teardown releases the routes.
10337            let holdouts = forwarding
10338                .endpoint_drain_holdouts(target.endpoint)
10339                .unwrap_or_default();
10340            warn_drain_timed_out(
10341                &spec.module_id,
10342                drain_timeout,
10343                reason,
10344                &holdouts,
10345                &gauge_trace,
10346            );
10347        }
10348        crate::control::send_route_control_pushes(
10349            forwarding,
10350            routes,
10351            ClientControlPush::RouteClosed {
10352                module_id: spec.module_id.clone(),
10353                channels: Vec::new(),
10354                reason,
10355                drained,
10356                abandoned: target.abandoned_bindings.len() as u32,
10357                excluded_subscriptions: target.excluded_subscriptions,
10358                terminal: Some(terminal),
10359            },
10360        );
10361        wait_result?;
10362
10363        // `route.closed` has now been sent unconditionally above. From here the
10364        // remaining steps are cleanup (route + module GOODBYE) rather than a
10365        // promise the client is waiting on, but a lock-poisoned
10366        // `release_module_endpoint_routes` would otherwise skip the module
10367        // GOODBYE silently too -- send it before propagating the error.
10368        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10369            Ok(routes) => routes,
10370            Err(err) => {
10371                warn!(
10372                    module_id = %spec.module_id,
10373                    ?reason,
10374                    error = %err,
10375                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10376                );
10377                send_module_goodbye(&spec.module_id, forwarding, target);
10378                return Err(SuperviseError::Forwarding(err));
10379            }
10380        };
10381        let route_goodbye_count = released_routes.len();
10382        send_route_goodbyes(forwarding, released_routes);
10383        send_module_goodbye(&spec.module_id, forwarding, target);
10384
10385        // The drain's happy path was previously silent: every emission above is
10386        // best-effort with only its failure arm logged, so "were consumers told"
10387        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10388        // hang where the open question was exactly whether teardown notice went
10389        // out). One summary line makes that class decidable in one grep.
10390        info!(
10391            module_id = %spec.module_id,
10392            ?reason,
10393            routes_notified,
10394            route_goodbyes = route_goodbye_count,
10395            abandoned_reservations = target.abandoned_bindings.len(),
10396            excluded_subscriptions = target.excluded_subscriptions,
10397            drained,
10398            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10399        );
10400    }
10401
10402    Ok(StopNotice::SentOverConnection)
10403}
10404
10405/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10406/// the only slot a plain (non-swap) spawn can register into.
10407async fn wait_for_registration_after_reload(
10408    registry: &Registry,
10409    module_id: &str,
10410    snapshot: &SharedSnapshot,
10411    child: &mut SupervisedChild,
10412    wait: Duration,
10413) -> Result<RegistrationWaitOutcome, SuperviseError> {
10414    wait_for_slot_registration(
10415        registry,
10416        crate::registry::RegistrationSlot::Active(module_id),
10417        module_id,
10418        snapshot,
10419        child,
10420        wait,
10421    )
10422    .await
10423}
10424
10425/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10426///
10427/// Keyed on the slot rather than the bare module id because during a swap the
10428/// id's active slot is already held by the incumbent: an id-keyed wait would
10429/// report the incumbent's registration as the candidate's and a candidate that
10430/// never registers would look registered. A swap candidate waits on
10431/// `crate::registry::RegistrationSlot::Candidate`.
10432async fn wait_for_slot_registration(
10433    registry: &Registry,
10434    slot: crate::registry::RegistrationSlot<'_>,
10435    module_id: &str,
10436    snapshot: &SharedSnapshot,
10437    child: &mut SupervisedChild,
10438    wait: Duration,
10439) -> Result<RegistrationWaitOutcome, SuperviseError> {
10440    let deadline = Instant::now() + wait;
10441    loop {
10442        if registry
10443            .registration(slot)
10444            .map_err(SuperviseError::Registry)?
10445            .is_some()
10446        {
10447            return Ok(RegistrationWaitOutcome::Registered);
10448        }
10449
10450        let now = Instant::now();
10451        if now >= deadline {
10452            return Ok(RegistrationWaitOutcome::TimedOut);
10453        }
10454        let remaining = deadline.saturating_duration_since(now);
10455        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10456
10457        tokio::select! {
10458            wait_result = child.wait() => {
10459                let status = wait_result.map_err(|source| SuperviseError::Wait {
10460                    module_id: module_id.to_string(),
10461                    source,
10462                })?;
10463                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10464                    snapshot,
10465                    child,
10466                    &status,
10467                )));
10468            }
10469            _ = sleep(poll) => {}
10470        }
10471    }
10472}
10473
10474fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10475    // A replacement process that exits before HELLO did not provide service, even
10476    // if it used status 0. Count it against the restart cap as a new-binary failure.
10477    if exit_report.kind != ExitKind::DeliberateSeverance {
10478        exit_report.kind = ExitKind::Crash;
10479    }
10480    exit_report
10481}
10482
10483async fn handle_reload_child_registration_failure(
10484    spec: &ModuleSpec,
10485    runtime: &SupervisorRuntimeConfig,
10486    registry: &Registry,
10487    process_liveness: &SupervisorProcessLiveness,
10488    snapshot: &SharedSnapshot,
10489    _child: &mut Option<SupervisedChild>,
10490    failure: ReloadRegistrationFailure,
10491) -> Result<(), SuperviseError> {
10492    let ReloadRegistrationFailure {
10493        exit_report,
10494        reason,
10495    } = failure;
10496    match on_child_exit(
10497        spec,
10498        runtime.restart_policy,
10499        registry,
10500        snapshot,
10501        &runtime.terminal_ring,
10502        &runtime.spawn_events,
10503        &runtime.child_roster,
10504        exit_report,
10505    )
10506    .await
10507    {
10508        NextAction::Stop {
10509            registration_released,
10510        } => {
10511            if registration_released {
10512                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10513            }
10514        }
10515        NextAction::Restart { schedule } => {
10516            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10517                schedule.delay
10518            });
10519            if let Some(schedule) = schedule {
10520                log_crash_respawn(&spec.module_id, schedule);
10521            }
10522            schedule_respawn(
10523                runtime,
10524                snapshot,
10525                &spec.module_id,
10526                delay,
10527                RespawnKind::Spawn,
10528            )?;
10529        }
10530    }
10531    Err(SuperviseError::ReloadFailed {
10532        module_id: spec.module_id.clone(),
10533        reason,
10534    })
10535}
10536
10537async fn handle_reload_spawn_failure(
10538    spec: &ModuleSpec,
10539    runtime: &SupervisorRuntimeConfig,
10540    process_liveness: &SupervisorProcessLiveness,
10541    snapshot: &SharedSnapshot,
10542    _child: &mut Option<SupervisedChild>,
10543    reason: String,
10544) -> Result<(), SuperviseError> {
10545    let now = Instant::now();
10546    let mut schedule = None;
10547    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10548        clear_current_process_facts(state);
10549        if state.enabled {
10550            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10551            state.state = if schedule.is_some() {
10552                ModuleState::Restarting
10553            } else {
10554                ModuleState::Failed
10555            };
10556        } else {
10557            state.state = ModuleState::Disabled;
10558        }
10559    })?;
10560    if let Some(schedule) = schedule {
10561        schedule_respawn(
10562            runtime,
10563            snapshot,
10564            &spec.module_id,
10565            schedule.delay,
10566            RespawnKind::Spawn,
10567        )?;
10568    } else {
10569        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10570    }
10571    Err(SuperviseError::ReloadFailed {
10572        module_id: spec.module_id.clone(),
10573        reason,
10574    })
10575}
10576
10577fn control_flags() -> Flags {
10578    Flags::new(false, Priority::Passive, false)
10579}
10580
10581#[allow(clippy::too_many_arguments)]
10582async fn drain_optional_child(
10583    module_id: &str,
10584    protocol: ModuleProtocol,
10585    stop_notice: StopNotice,
10586    registry: &Registry,
10587    forwarding: Option<&ForwardingTable>,
10588    snapshot: &SharedSnapshot,
10589    terminal_ring: &Arc<Mutex<TerminalRing>>,
10590    spawn_events: &SpawnEventFeed,
10591    child: &mut Option<SupervisedChild>,
10592    drain_timeout: Duration,
10593    final_state: ModuleState,
10594    enabled: Option<bool>,
10595) -> Result<(), SuperviseError> {
10596    if let Some(child) = child.take() {
10597        drain_child_to_state(
10598            module_id,
10599            protocol,
10600            stop_notice,
10601            registry,
10602            forwarding,
10603            snapshot,
10604            terminal_ring,
10605            spawn_events,
10606            child,
10607            drain_timeout,
10608            final_state,
10609            enabled,
10610        )
10611        .await
10612    } else {
10613        update_snapshot(snapshot, Some(module_id), |state| {
10614            state.state = final_state;
10615            if let Some(enabled) = enabled {
10616                state.enabled = enabled;
10617            }
10618            clear_current_process_facts(state);
10619        })?;
10620        release_dead_registration(registry, forwarding, snapshot, module_id).await
10621    }
10622}
10623
10624#[allow(clippy::too_many_arguments)]
10625async fn drain_child_to_state(
10626    module_id: &str,
10627    _protocol: ModuleProtocol,
10628    stop_notice: StopNotice,
10629    registry: &Registry,
10630    forwarding: Option<&ForwardingTable>,
10631    snapshot: &SharedSnapshot,
10632    terminal_ring: &Arc<Mutex<TerminalRing>>,
10633    spawn_events: &SpawnEventFeed,
10634    mut child: SupervisedChild,
10635    drain_timeout: Duration,
10636    final_state: ModuleState,
10637    enabled: Option<bool>,
10638) -> Result<(), SuperviseError> {
10639    let protocol = child.protocol;
10640    update_snapshot(snapshot, Some(module_id), |state| {
10641        state.state = ModuleState::Draining;
10642        state.draining_to_replace = final_state == ModuleState::Restarting;
10643        if let Some(enabled) = enabled {
10644            state.enabled = enabled;
10645        }
10646    })?;
10647
10648    // The wait below is the same budget in every case; what differs is
10649    // whether anything has ASKED the child to stop before it starts. Only a
10650    // forwarding drain that reached the module's registered connection has
10651    // (`module.draining`, then a module GOODBYE). Every other child was told
10652    // nothing: a `protocol: "none"` module, which never registers; a subc
10653    // module spawned moments ago that has not sent HELLO yet; or a stop that
10654    // runs no forwarding drain. Without a signal the budget is only a delay
10655    // in front of SIGKILL -- and the not-yet-registered child is the worst
10656    // case, because it registers into a module that is already draining,
10657    // is never told, and is killed while healthy.
10658    if stop_notice != StopNotice::SentOverConnection {
10659        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10660            info!(
10661                module_id,
10662                pid = child.pid,
10663                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10664                "module has no connection yet; requesting stop by signal"
10665            );
10666        }
10667        request_graceful_stop(module_id, &child);
10668    }
10669
10670    let exit_report = match timeout(drain_timeout, child.wait()).await {
10671        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10672        Ok(Err(source)) => {
10673            fail_after_wait_error(
10674                module_id,
10675                snapshot,
10676                terminal_ring,
10677                spawn_events,
10678                format!("failed to wait during drain: {source}"),
10679            );
10680            return Err(SuperviseError::Wait {
10681                module_id: module_id.to_string(),
10682                source,
10683            });
10684        }
10685        Err(_) => {
10686            // Mirror the sibling arm above: state is already `Draining`, and an
10687            // error propagated from here would strand it there -- a state
10688            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10689            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10690            // `Failed` before `?` keeps the module operator-visible and
10691            // revivable. Trigger is an ESRCH race (process exits between the
10692            // drain timeout firing and the kill) or a post-kill wait failure
10693            // (issue #34).
10694            //
10695            // Logged because the kill is otherwise visible only as signal 9 in
10696            // the terminal ring, and the budget it follows can be long enough
10697            // that consumers see a stretch of refusals with no stated cause.
10698            warn!(
10699                module_id,
10700                pid = child.pid,
10701                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10702                reason = ?final_state,
10703                ?stop_notice,
10704                "drain budget expired before the module exited; killing it"
10705            );
10706            child.start_kill().map_err(|source| {
10707                fail_after_wait_error(
10708                    module_id,
10709                    snapshot,
10710                    terminal_ring,
10711                    spawn_events,
10712                    format!("failed to kill during drain: {source}"),
10713                );
10714                SuperviseError::Kill {
10715                    module_id: module_id.to_string(),
10716                    source,
10717                }
10718            })?;
10719            let status = child.wait().await.map_err(|source| {
10720                fail_after_wait_error(
10721                    module_id,
10722                    snapshot,
10723                    terminal_ring,
10724                    spawn_events,
10725                    format!("failed to wait after drain kill: {source}"),
10726                );
10727                SuperviseError::Wait {
10728                    module_id: module_id.to_string(),
10729                    source,
10730                }
10731            })?;
10732            classify_reaped_child_exit(snapshot, &child, &status)
10733        }
10734    };
10735
10736    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10737    publish_after_terminal_record(
10738        module_id,
10739        || {
10740            record_terminal_with_detail(
10741                module_id,
10742                terminal_ring,
10743                spawn_events,
10744                &exit_report,
10745                terminal_disposition(final_state),
10746                detail,
10747            )
10748        },
10749        || {
10750            update_snapshot(snapshot, Some(module_id), |state| {
10751                state.state = final_state;
10752                if let Some(enabled) = enabled {
10753                    state.enabled = enabled;
10754                }
10755                clear_current_process_facts(state);
10756                state.last_exit = Some(exit_report.clone());
10757                if exit_report.kind == ExitKind::DeliberateSeverance {
10758                    state.lifetime_restarts += 1;
10759                }
10760            })
10761        },
10762    )?;
10763    child.drain_stderr(module_id).await;
10764
10765    release_dead_registration(registry, forwarding, snapshot, module_id).await
10766}
10767
10768/// Ask a child that nothing else has asked to stop, by signal.
10769///
10770/// A registered subc module is asked over its own connection: the drain sends
10771/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10772/// module GOODBYE, and the module stops itself. A module that speaks no subc
10773/// wire receives none of that, and neither does a subc module that has not
10774/// registered yet, so for them the drain budget would be pure delay in front of
10775/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10776/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10777/// into a recovery on the next start.
10778///
10779/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10780/// rule rather than an optimisation: that module's graceful stop is already
10781/// running by the time its child is drained, and a signal would race it.
10782///
10783/// Best-effort by construction. A child that has already exited is the ordinary
10784/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10785/// failure is logged at debug and the wait-then-kill below still decides the
10786/// outcome.
10787#[cfg(unix)]
10788fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10789    let Some(pid) = child
10790        .id()
10791        .and_then(|pid| i32::try_from(pid).ok())
10792        .and_then(rustix::process::Pid::from_raw)
10793    else {
10794        debug!(
10795            module_id,
10796            "no pid to signal for teardown; falling through to the drain wait"
10797        );
10798        return;
10799    };
10800    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10801        Ok(()) => debug!(
10802            module_id,
10803            "sent SIGTERM to a module nothing else asked to stop"
10804        ),
10805        Err(err) => debug!(
10806            module_id,
10807            error = %err,
10808            "SIGTERM to module failed; the drain wait and kill still apply"
10809        ),
10810    }
10811}
10812
10813/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10814/// Windows does offer need cooperation this supervisor cannot assume: a console
10815/// control event requires sharing a console with the child, and `WM_CLOSE`
10816/// requires the child to pump a message loop. A supervised server process does
10817/// neither, so there is nothing to send and teardown is the wait followed by the
10818/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10819/// the thing `protocol: "none"` exists to avoid.
10820#[cfg(not(unix))]
10821fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10822    debug!(
10823        module_id,
10824        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10825    );
10826}
10827
10828fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10829    match final_state {
10830        ModuleState::Stopped => TerminalDisposition::Stopped,
10831        ModuleState::Disabled => TerminalDisposition::Disabled,
10832        ModuleState::Restarting => TerminalDisposition::Restarting,
10833        ModuleState::Failed => TerminalDisposition::Failed,
10834        ModuleState::Starting
10835        | ModuleState::Running
10836        | ModuleState::Unresponsive
10837        | ModuleState::Draining => {
10838            unreachable!("terminal exits only finish in terminal or restarting states")
10839        }
10840    }
10841}
10842
10843/// Release a reaped child's registration before allowing another spawn.
10844///
10845/// EOF is not a process-lifetime signal: an inherited socket can stay open
10846/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10847/// reading EOF. After the normal release grace, request connection close (which
10848/// cancels both reads and dispatch), then allow one more release grace for the
10849/// connection guard's forwarding cleanup. Never evict a different connection.
10850async fn release_dead_registration(
10851    registry: &Registry,
10852    forwarding: Option<&ForwardingTable>,
10853    snapshot: &SharedSnapshot,
10854    module_id: &str,
10855) -> Result<(), SuperviseError> {
10856    let result = async {
10857        let registration = registry
10858            .get_module(module_id)
10859            .map_err(SuperviseError::Registry)?;
10860        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10861            Ok(()) => return Ok(()),
10862            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10863            Err(err) => return Err(err),
10864        }
10865        let pid = lock_snapshot(snapshot)?.reaped_pid;
10866        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10867            warn!(
10868                module_id,
10869                pid,
10870                connection_id = registration.connection_id.get(),
10871                "reaped module registration outlived release grace; closing dead connection"
10872            );
10873            forwarding.request_connection_close(
10874                registration.connection_id,
10875                CloseReason::new(
10876                    "supervised_process_reaped",
10877                    format!("module '{module_id}' pid {pid} exited"),
10878                ),
10879            );
10880            wait_for_slot_registration_release(
10881                registry,
10882                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10883                REGISTRY_RELEASE_TIMEOUT,
10884            )
10885            .await?;
10886        }
10887        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10888    }
10889    .await;
10890    if let Err(err) = &result {
10891        fail_snapshot(snapshot, Some(module_id), None);
10892        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10893    }
10894    result
10895}
10896
10897/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10898/// plain stop or restart waits for before it spawns a replacement.
10899async fn wait_for_registration_release(
10900    registry: &Registry,
10901    module_id: &str,
10902    wait: Duration,
10903) -> Result<(), SuperviseError> {
10904    wait_for_slot_registration_release(
10905        registry,
10906        crate::registry::RegistrationSlot::Active(module_id),
10907        wait,
10908    )
10909    .await
10910}
10911
10912/// Wait for the registration in `slot` to go away.
10913///
10914/// Keyed on the slot rather than the bare module id because a successful swap
10915/// never empties the id's active slot (the promoted candidate is in it), so an
10916/// id-keyed wait for the incumbent's release would always time out. Draining a
10917/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10918/// incumbent's connection instead.
10919async fn wait_for_slot_registration_release(
10920    registry: &Registry,
10921    slot: crate::registry::RegistrationSlot<'_>,
10922    wait: Duration,
10923) -> Result<(), SuperviseError> {
10924    let deadline = Instant::now() + wait;
10925    let mut release_events = registration_release_events().subscribe();
10926    let still_active = |registration: &crate::registry::ModuleRegistration| {
10927        SuperviseError::RegistrationStillActive {
10928            module_id: registration.manifest.module_id.clone(),
10929            waited: wait,
10930        }
10931    };
10932    loop {
10933        let _observed_generation = *release_events.borrow_and_update();
10934        let Some(registration) = registry
10935            .registration(slot)
10936            .map_err(SuperviseError::Registry)?
10937        else {
10938            return Ok(());
10939        };
10940
10941        let now = Instant::now();
10942        if now >= deadline {
10943            return Err(still_active(&registration));
10944        }
10945
10946        let remaining = deadline.saturating_duration_since(now);
10947        match timeout(remaining, release_events.changed()).await {
10948            Ok(Ok(())) | Ok(Err(_)) => {}
10949            Err(_) => return Err(still_active(&registration)),
10950        }
10951    }
10952}
10953
10954#[cfg(test)]
10955mod slot_registration_wait_tests {
10956    use super::*;
10957    use crate::registry::{ConnectionId, RegistrationSlot};
10958    use subc_protocol::manifest::ModuleManifest;
10959
10960    #[tokio::test]
10961    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10962        let registry = Arc::new(Registry::default());
10963        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10964        let runtime = supervisor.runtime_config();
10965        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10966        let spec = ModuleSpec {
10967            module_id: "enable-stale-registration".to_string(),
10968            program: PathBuf::from("/missing/enable-retry-test"),
10969            args: Vec::new(),
10970            env: Vec::new(),
10971            reserved: false,
10972            reserved_prefixes: Vec::new(),
10973            protocol: ModuleProtocol::Subc,
10974            overlap: Default::default(),
10975        };
10976        let connection = ConnectionId::new(90);
10977        registry
10978            .register_with_control_ops(
10979                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10980                1,
10981                connection,
10982                Vec::new(),
10983            )
10984            .unwrap();
10985        let mut child = None;
10986        let err = set_child_enabled(
10987            &spec,
10988            &runtime,
10989            &registry,
10990            &supervisor.process_liveness,
10991            &snapshot,
10992            &mut child,
10993            true,
10994        )
10995        .await
10996        .unwrap_err();
10997        assert!(matches!(
10998            err,
10999            SuperviseError::RegistrationStillActive { .. }
11000        ));
11001        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11002        assert!(child.is_none());
11003        registry.deregister_connection(connection).unwrap();
11004        let err = set_child_enabled(
11005            &spec,
11006            &runtime,
11007            &registry,
11008            &supervisor.process_liveness,
11009            &snapshot,
11010            &mut child,
11011            true,
11012        )
11013        .await
11014        .unwrap_err();
11015        assert!(
11016            matches!(err, SuperviseError::Spawn { .. }),
11017            "second enable must attempt a spawn: {err}"
11018        );
11019        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11020    }
11021
11022    const INCUMBENT: u64 = 1;
11023    const CANDIDATE: u64 = 2;
11024
11025    fn swapped_registry() -> Arc<Registry> {
11026        let registry = Arc::new(Registry::default());
11027        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11028        registry
11029            .register_with_control_ops(
11030                manifest.clone(),
11031                1,
11032                ConnectionId::new(INCUMBENT),
11033                Vec::new(),
11034            )
11035            .unwrap();
11036        registry
11037            .register_candidate_with_control_ops(
11038                manifest,
11039                1,
11040                ConnectionId::new(CANDIDATE),
11041                Vec::new(),
11042            )
11043            .unwrap();
11044        registry
11045    }
11046
11047    /// After a promotion the id's active slot is held by the new process, so an
11048    /// id-keyed wait for the incumbent's release can never succeed; the
11049    /// connection-keyed wait completes as soon as the incumbent deregisters.
11050    #[tokio::test]
11051    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11052        let registry = swapped_registry();
11053        registry.promote_candidate("m").unwrap().unwrap();
11054
11055        assert!(matches!(
11056            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11057            Err(SuperviseError::RegistrationStillActive { .. })
11058        ));
11059
11060        // Still held while the incumbent's connection has not deregistered.
11061        assert!(matches!(
11062            wait_for_slot_registration_release(
11063                &registry,
11064                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11065                Duration::from_millis(50),
11066            )
11067            .await,
11068            Err(SuperviseError::RegistrationStillActive { .. })
11069        ));
11070
11071        let releaser = Arc::clone(&registry);
11072        let release = tokio::spawn(async move {
11073            sleep(Duration::from_millis(20)).await;
11074            releaser
11075                .deregister_connection(ConnectionId::new(INCUMBENT))
11076                .unwrap();
11077            notify_registration_release();
11078        });
11079        wait_for_slot_registration_release(
11080            &registry,
11081            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11082            Duration::from_secs(5),
11083        )
11084        .await
11085        .expect("the incumbent's own registration is released");
11086        release.await.unwrap();
11087        assert!(registry.get_module("m").unwrap().is_some());
11088    }
11089
11090    /// The candidate slot is waited on separately from the active slot: the
11091    /// incumbent's registration neither holds up nor stands in for it.
11092    #[tokio::test]
11093    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11094        let registry = swapped_registry();
11095        assert!(matches!(
11096            wait_for_slot_registration_release(
11097                &registry,
11098                RegistrationSlot::Candidate("m"),
11099                Duration::from_millis(50),
11100            )
11101            .await,
11102            Err(SuperviseError::RegistrationStillActive { .. })
11103        ));
11104        registry
11105            .deregister_connection(ConnectionId::new(CANDIDATE))
11106            .unwrap();
11107        wait_for_slot_registration_release(
11108            &registry,
11109            RegistrationSlot::Candidate("m"),
11110            Duration::from_millis(50),
11111        )
11112        .await
11113        .expect("a candidate slot with no candidate is released");
11114        assert!(registry
11115            .registration(RegistrationSlot::Active("m"))
11116            .unwrap()
11117            .is_some());
11118    }
11119}
11120
11121fn classify_exit(status: &ExitStatus) -> ExitReport {
11122    ExitReport {
11123        kind: if status.success() {
11124            ExitKind::Clean
11125        } else {
11126            ExitKind::Crash
11127        },
11128        code: status.code(),
11129        signal: exit_signal(status),
11130        at_ms: unix_ms_now(),
11131    }
11132}
11133
11134/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11135/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11136/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11137/// disposition still must be `Failed` so the terminal ring is not silently missing
11138/// an entry, matching what `fail_snapshot` records for this same arm.
11139fn wait_error_exit_report() -> ExitReport {
11140    ExitReport {
11141        kind: ExitKind::Crash,
11142        code: None,
11143        signal: None,
11144        at_ms: unix_ms_now(),
11145    }
11146}
11147
11148#[cfg(unix)]
11149fn exit_signal(status: &ExitStatus) -> Option<i32> {
11150    use std::os::unix::process::ExitStatusExt;
11151
11152    status.signal()
11153}
11154
11155#[cfg(not(unix))]
11156fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11157    None
11158}
11159
11160/// Give an operator-touched module its full crash budget back.
11161///
11162/// Named for the counter it used to zero; it now empties the in-window ring,
11163/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11164/// ledger of what happened survives every operator action.
11165fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11166    update_snapshot(snapshot, Some(module_id), |state| {
11167        state.clear_crash_restarts();
11168    })
11169}
11170
11171fn set_running(
11172    snapshot: &SharedSnapshot,
11173    child: &SupervisedChild,
11174    module_id: &str,
11175    spawn_events: &SpawnEventFeed,
11176) -> Result<(), SuperviseError> {
11177    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11178        module_id: Some(module_id.to_string()),
11179    })?;
11180    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11181    if std::mem::take(&mut state.coalesced_restart_pending) {
11182        let generation = state.spawn_generation;
11183        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11184    }
11185    state.drain_disposition_detail = None;
11186    state.spawn_failure = None;
11187    state.waiting_for_previous_pid = None;
11188    state.waiting_for_previous_detail = None;
11189    // Every caller of this is a plain spawn, which always uses the primary key;
11190    // a promoted swap candidate sets the flag itself after this returns.
11191    state.in_alternate_slot = false;
11192    state.configuration_updated_since_spawn = false;
11193    state.spawned_protocol = Some(child.protocol);
11194    state.state = ModuleState::Running;
11195    state.enabled = true;
11196    state.process_alive = true;
11197    state.pid = child.id();
11198    #[cfg(windows)]
11199    {
11200        state.pipe_consumed = child
11201            .nonce_delivery
11202            .as_ref()
11203            .map(|delivery| delivery.consumed());
11204    }
11205    #[cfg(target_os = "macos")]
11206    {
11207        state.report_ready = Some(Arc::clone(&child.report_ready));
11208    }
11209    state.spawned_at_ms = Some(child.spawned_at_ms);
11210    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11211    state.spawned_from = Some(child.spawned_from.clone());
11212    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11213    state.spawned_file_identity = child.spawned_file_identity.clone();
11214    state.process_start_time = child.process_start_time;
11215    Ok(())
11216}
11217
11218fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11219    state.waiting_for_previous_pid = None;
11220    state.waiting_for_previous_detail = None;
11221    state.process_alive = false;
11222    state.spawned_protocol = None;
11223    state.pid = None;
11224    #[cfg(windows)]
11225    {
11226        state.pipe_consumed = None;
11227    }
11228    #[cfg(target_os = "macos")]
11229    {
11230        state.report_ready = None;
11231    }
11232    state.spawned_at_ms = None;
11233    state.exec_outcome = None;
11234    state.spawned_from = None;
11235    state.launch_spec_digest = None;
11236    state.spawned_file_identity = None;
11237    state.process_start_time = None;
11238    state.deliberate_severance = None;
11239}
11240
11241#[cfg(test)]
11242fn record_deliberate_severance(
11243    snapshot: &SharedSnapshot,
11244    identity: ProcessIdentity,
11245) -> Result<(), SuperviseError> {
11246    update_snapshot(snapshot, None, |state| {
11247        state.deliberate_severance = Some(identity);
11248    })
11249}
11250
11251fn apply_deliberate_severance_marker(
11252    snapshot: &SharedSnapshot,
11253    exited_identity: Option<ProcessIdentity>,
11254    mut exit_report: ExitReport,
11255) -> ExitReport {
11256    let marker = lock_snapshot(snapshot)
11257        .ok()
11258        .and_then(|mut state| state.deliberate_severance.take());
11259    if marker.is_some() && marker == exited_identity {
11260        exit_report.kind = ExitKind::DeliberateSeverance;
11261    }
11262    exit_report
11263}
11264
11265fn classify_reaped_child_exit(
11266    snapshot: &SharedSnapshot,
11267    child: &SupervisedChild,
11268    status: &ExitStatus,
11269) -> ExitReport {
11270    let _ = update_snapshot(snapshot, None, |state| {
11271        state.reaped_pid = Some(child.pid);
11272        state.spawn_failure = child.spawn_failure.clone();
11273    });
11274    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11275}
11276
11277fn fail_snapshot(
11278    snapshot: &SharedSnapshot,
11279    module_id: Option<&str>,
11280    last_exit: Option<ExitReport>,
11281) {
11282    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11283        state.state = ModuleState::Failed;
11284        clear_current_process_facts(state);
11285        if let Some(last_exit) = last_exit {
11286            state.last_exit = Some(last_exit);
11287        }
11288    }) {
11289        error!(error = %err, "failed to mark supervisor state failed");
11290    }
11291}
11292
11293fn update_snapshot(
11294    snapshot: &SharedSnapshot,
11295    module_id: Option<&str>,
11296    update: impl FnOnce(&mut SupervisorSnapshot),
11297) -> Result<(), SuperviseError> {
11298    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11299        module_id: module_id.map(ToOwned::to_owned),
11300    })?;
11301    update(&mut state);
11302    Ok(())
11303}
11304
11305const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11306
11307fn lock_snapshot_for_control<'a>(
11308    snapshot: &'a SharedSnapshot,
11309    module_id: &str,
11310    caller: &'static str,
11311) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11312    let started_at = Instant::now();
11313    let guard = lock_snapshot(snapshot)?;
11314    let waited = started_at.elapsed();
11315    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11316        warn!(
11317            module_id = %module_id,
11318            waited_ms = waited.as_millis() as u64,
11319            caller = %caller,
11320            "slow snapshot lock"
11321        );
11322    }
11323    Ok(guard)
11324}
11325
11326fn lock_snapshot(
11327    snapshot: &SharedSnapshot,
11328) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11329    snapshot
11330        .lock()
11331        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11332}
11333
11334#[cfg(test)]
11335mod terminal_history_tests {
11336    use std::{
11337        path::PathBuf,
11338        sync::Arc,
11339        time::{Duration, Instant},
11340    };
11341
11342    use tokio::{sync::mpsc, time::sleep};
11343
11344    use super::{
11345        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11346        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11347        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11348        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11349        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11350        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11351        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11352        SupervisorHealthStatus, SupervisorSnapshot,
11353    };
11354    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11355    // use for their own wall-clock deadlines: crash-restart instants must be on
11356    // the same clock the production code stamps them with, which is tokio's (and
11357    // is what `start_paused` tests can move).
11358    use super::Instant as ClockInstant;
11359    use crate::{
11360        registry::Registry,
11361        terminal_ring::{TerminalRing, TerminalRingConfig},
11362    };
11363    use std::sync::Mutex;
11364    use subc_control::TerminalDisposition;
11365
11366    /// See the twin in `control.rs` for why this derives the path from
11367    /// `current_exe()` and why the existence check is here: `--lib` alone does
11368    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11369    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11370    pub(super) fn fake_aft_stub_path() -> PathBuf {
11371        let mut path = std::env::current_exe().expect("current_exe available in tests");
11372        path.pop();
11373        path.pop();
11374        path.push(if cfg!(windows) {
11375            "fake-aft-stub.exe"
11376        } else {
11377            "fake-aft-stub"
11378        });
11379        assert!(
11380            path.exists(),
11381            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11382             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11383            path.display()
11384        );
11385        path
11386    }
11387
11388    #[test]
11389    fn reserved_never_spawned_refuses_every_hello() {
11390        // The canary hole: a reserved id whose module has never spawned had NO
11391        // gate entry and admitted anyone -- the reservation protected the nonce
11392        // holder, not the NAME. Now the entry is present with no legitimate
11393        // holder and refuses all comers.
11394        let supervisor = SupervisorHandle::default();
11395        supervisor.apply_identity_configuration(&ModuleSpec {
11396            module_id: "never-spawned".to_string(),
11397            program: PathBuf::from("/usr/bin/false"),
11398            args: Vec::new(),
11399            env: Vec::new(),
11400            reserved: true,
11401            reserved_prefixes: Vec::new(),
11402            protocol: ModuleProtocol::Subc,
11403            overlap: Default::default(),
11404        });
11405        assert!(
11406            supervisor
11407                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11408                .is_some(),
11409            "forged nonce must refuse on a reserved never-spawned id"
11410        );
11411        assert!(
11412            supervisor
11413                .reserved_hello_rejection("never-spawned", None)
11414                .is_some(),
11415            "absent nonce must refuse on a reserved never-spawned id"
11416        );
11417        // And a real spawn nonce minted later admits exactly that nonce.
11418        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11419        supervisor.apply_identity_configuration(&ModuleSpec {
11420            module_id: "never-spawned".to_string(),
11421            program: PathBuf::from("/usr/bin/false"),
11422            args: Vec::new(),
11423            env: Vec::new(),
11424            reserved: true,
11425            reserved_prefixes: Vec::new(),
11426            protocol: ModuleProtocol::Subc,
11427            overlap: Default::default(),
11428        });
11429        assert!(supervisor
11430            .reserved_hello_rejection("never-spawned", Some("minted"))
11431            .is_none());
11432        assert!(supervisor
11433            .reserved_hello_rejection("never-spawned", Some("forged"))
11434            .is_some());
11435    }
11436
11437    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11438    /// happened, which is what "spent budget" looks like to every reader.
11439    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11440        let now = ClockInstant::now();
11441        for _ in 0..count {
11442            state.crash_restarts.push_back(now);
11443        }
11444    }
11445
11446    /// Age the oldest recorded restart out of `window`, standing in for the hours
11447    /// that would otherwise have to pass. Injecting the instant is the point: a
11448    /// test that slept a real window would take ten minutes and still prove less.
11449    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11450        let aged = state
11451            .crash_restarts
11452            .front()
11453            .expect("a crash restart must be recorded before it can be aged")
11454            .checked_sub(window + Duration::from_secs(1))
11455            .expect("the test clock is far enough from its origin to age an instant");
11456        state.crash_restarts[0] = aged;
11457    }
11458
11459    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11460        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11461        seed_crash_restarts(&mut state, count);
11462        state
11463    }
11464
11465    #[test]
11466    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11467        let policy = RestartPolicy::new(3, Duration::ZERO);
11468        let now = ClockInstant::now();
11469        assert!(daemon_will_restart(
11470            &mut snapshot_with_restarts(true, 2),
11471            &policy,
11472            now
11473        ));
11474        assert!(!daemon_will_restart(
11475            &mut snapshot_with_restarts(true, 3),
11476            &policy,
11477            now
11478        ));
11479        assert!(!daemon_will_restart(
11480            &mut snapshot_with_restarts(false, 0),
11481            &policy,
11482            now
11483        ));
11484    }
11485
11486    #[test]
11487    fn crash_restart_backoff_escalates_with_in_window_count() {
11488        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11489            .with_max_backoff(Duration::from_secs(30));
11490        let now = ClockInstant::now();
11491        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11492        let schedules = (0..4)
11493            .map(|_| {
11494                state
11495                    .next_crash_restart(&policy, now)
11496                    .expect("the test policy allows four crash restarts")
11497            })
11498            .collect::<Vec<_>>();
11499
11500        assert_eq!(
11501            schedules
11502                .iter()
11503                .map(|schedule| schedule.restart_in_window)
11504                .collect::<Vec<_>>(),
11505            vec![0, 1, 2, 3]
11506        );
11507        assert_eq!(
11508            schedules
11509                .iter()
11510                .map(|schedule| schedule.delay)
11511                .collect::<Vec<_>>(),
11512            vec![
11513                Duration::from_millis(100),
11514                Duration::from_secs(1),
11515                Duration::from_secs(10),
11516                Duration::from_secs(30),
11517            ]
11518        );
11519    }
11520
11521    #[test]
11522    fn crash_restart_backoff_resets_after_ring_clear() {
11523        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11524        let now = ClockInstant::now();
11525        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11526        assert_eq!(
11527            state.next_crash_restart(&policy, now).unwrap().delay,
11528            Duration::from_millis(100)
11529        );
11530        assert_eq!(
11531            state.next_crash_restart(&policy, now).unwrap().delay,
11532            Duration::from_secs(1)
11533        );
11534
11535        state.clear_crash_restarts();
11536        let schedule = state
11537            .next_crash_restart(&policy, now)
11538            .expect("a cleared ring must allow another restart");
11539        assert_eq!(schedule.restart_in_window, 0);
11540        assert_eq!(schedule.delay, Duration::from_millis(100));
11541    }
11542
11543    #[test]
11544    fn crash_restart_backoff_ignores_aged_restarts() {
11545        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11546        let now = ClockInstant::now();
11547        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11548        state
11549            .next_crash_restart(&policy, now)
11550            .expect("the first restart is allowed");
11551        state
11552            .next_crash_restart(&policy, now)
11553            .expect("the second restart is allowed");
11554        state.crash_restarts[0] = now
11555            .checked_sub(policy.window + Duration::from_secs(1))
11556            .expect("the fake clock can age a restart past the window");
11557
11558        let schedule = state
11559            .next_crash_restart(&policy, now)
11560            .expect("an aged restart must release its slot");
11561        assert_eq!(schedule.restart_in_window, 1);
11562        assert_eq!(schedule.delay, Duration::from_secs(1));
11563        assert_eq!(state.crash_restarts.len(), 2);
11564    }
11565
11566    /// The budget is a rate: the same three spent restarts refuse a respawn
11567    /// while they are recent and allow one once they have aged past the window.
11568    /// Nothing about the module changed in between, which is the whole point.
11569    #[test]
11570    fn a_budget_spent_before_the_window_no_longer_refuses() {
11571        let policy = RestartPolicy::new(3, Duration::ZERO);
11572        let mut state = snapshot_with_restarts(true, 3);
11573        let now = ClockInstant::now();
11574        assert!(!daemon_will_restart(&mut state, &policy, now));
11575
11576        assert!(daemon_will_restart(
11577            &mut state,
11578            &policy,
11579            now + policy.window + Duration::from_secs(1)
11580        ));
11581        assert!(
11582            state.crash_restarts.is_empty(),
11583            "reading the budget must drop the instants that left the window"
11584        );
11585    }
11586
11587    fn module_with_recovery_snapshot(
11588        state: ModuleState,
11589        enabled: bool,
11590        restart_count: u32,
11591    ) -> SupervisedModule {
11592        let registry = Arc::new(Registry::default());
11593        let supervisor =
11594            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11595        let runtime = supervisor.runtime_config();
11596        let spec = ModuleSpec {
11597            module_id: "recovery-snapshot".to_string(),
11598            program: fake_aft_stub_path(),
11599            args: Vec::new(),
11600            env: Vec::new(),
11601            reserved: false,
11602            reserved_prefixes: Vec::new(),
11603            protocol: ModuleProtocol::Subc,
11604            overlap: Default::default(),
11605        };
11606        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11607        seed_crash_restarts(&mut snapshot, restart_count);
11608        // These tests read synthetic snapshots. A real child and monitor would
11609        // race those reads by replacing the requested state during startup.
11610        let (commands, _rx) = mpsc::channel(4);
11611        SupervisedModule {
11612            inner: Arc::new(SupervisedModuleInner {
11613                module_id: spec.module_id.clone(),
11614                registry,
11615                snapshot: Arc::new(Mutex::new(snapshot)),
11616                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11617                    spec,
11618                    health: runtime.health,
11619                })),
11620                stderr_ring: runtime.stderr_ring,
11621                terminal_ring: runtime.terminal_ring,
11622                commands,
11623                monitor: Mutex::new(None),
11624                restart_policy: runtime.restart_policy,
11625                effective_drain_timeout: runtime.effective_drain_timeout,
11626                provenance_probe: supervisor.provenance_probe.clone(),
11627            }),
11628        }
11629    }
11630
11631    #[cfg(target_os = "linux")]
11632    #[tokio::test]
11633    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11634        let supervisor =
11635            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11636                .with_cgroup_placement(None);
11637        let result = supervisor.spawn(ModuleSpec {
11638            module_id: "no-cgroup-placement".to_string(),
11639            program: fake_aft_stub_path(),
11640            args: Vec::new(),
11641            env: Vec::new(),
11642            reserved: false,
11643            reserved_prefixes: Vec::new(),
11644            protocol: ModuleProtocol::Subc,
11645            overlap: Default::default(),
11646        });
11647
11648        assert!(
11649            result.is_ok(),
11650            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11651        );
11652    }
11653
11654    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11655    async fn undecided_snapshot_uses_shared_restart_predicate() {
11656        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11657            .will_recover_after_connection_loss()
11658            .unwrap());
11659        assert!(
11660            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11661                .will_recover_after_connection_loss()
11662                .unwrap()
11663        );
11664    }
11665
11666    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11667    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11668        assert!(
11669            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11670                .will_recover_after_connection_loss()
11671                .unwrap()
11672        );
11673    }
11674
11675    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11676    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11677        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11678            .will_recover_after_connection_loss()
11679            .unwrap());
11680        assert!(
11681            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11682                .will_recover_after_connection_loss()
11683                .unwrap()
11684        );
11685    }
11686
11687    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11688    async fn warming_snapshot_is_limited_to_startup_phases() {
11689        for state in [
11690            ModuleState::Starting,
11691            ModuleState::Running,
11692            ModuleState::Restarting,
11693        ] {
11694            assert!(
11695                module_with_recovery_snapshot(state, true, 0)
11696                    .is_warming()
11697                    .unwrap(),
11698                "{state:?} should be warming"
11699            );
11700        }
11701        for state in [
11702            ModuleState::Unresponsive,
11703            ModuleState::Draining,
11704            ModuleState::Stopped,
11705            ModuleState::Failed,
11706            ModuleState::Disabled,
11707        ] {
11708            assert!(
11709                !module_with_recovery_snapshot(state, true, 0)
11710                    .is_warming()
11711                    .unwrap(),
11712                "{state:?} should not be warming"
11713            );
11714        }
11715    }
11716
11717    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11718    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11719        let registry = Arc::new(Registry::default());
11720        let supervisor =
11721            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11722        let module = supervisor
11723            .spawn(ModuleSpec {
11724                module_id: "terminal-history".to_string(),
11725                program: fake_aft_stub_path(),
11726                args: Vec::new(),
11727                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11728                reserved: false,
11729                reserved_prefixes: Vec::new(),
11730                protocol: ModuleProtocol::Subc,
11731                overlap: Default::default(),
11732            })
11733            .unwrap();
11734
11735        let deadline = Instant::now() + Duration::from_secs(5);
11736        loop {
11737            let history = module.terminal_history();
11738            if history.entries.len() == 2 {
11739                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11740                assert_eq!(history.dropped, 0);
11741                assert_eq!(
11742                    history
11743                        .entries
11744                        .iter()
11745                        .map(|entry| entry.exit_code)
11746                        .collect::<Vec<_>>(),
11747                    vec![Some(23), Some(23)]
11748                );
11749                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11750                return;
11751            }
11752            assert!(
11753                Instant::now() < deadline,
11754                "module did not retain two terminal exits: {history:?}"
11755            );
11756            sleep(Duration::from_millis(10)).await;
11757        }
11758    }
11759
11760    /// A disable issued while a crash respawn is still backing off must preempt
11761    /// that respawn: the operator's stop wins, the disable must not queue behind
11762    /// the backoff, and the module must never come back up afterwards.
11763    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11764    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11765        let backoff = Duration::from_secs(2);
11766        let supervisor = Supervisor::new_for_test(
11767            Arc::new(Registry::default()),
11768            RestartPolicy::new(10, backoff),
11769        );
11770        let module = supervisor
11771            .spawn(ModuleSpec {
11772                module_id: "disable-during-backoff".to_string(),
11773                program: fake_aft_stub_path(),
11774                args: Vec::new(),
11775                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11776                reserved: false,
11777                reserved_prefixes: Vec::new(),
11778                protocol: ModuleProtocol::Subc,
11779                overlap: Default::default(),
11780            })
11781            .unwrap();
11782
11783        // Wait for the first crash to put the module into its backoff window.
11784        let deadline = Instant::now() + Duration::from_secs(5);
11785        loop {
11786            if module.status().unwrap().state == ModuleState::Restarting {
11787                break;
11788            }
11789            assert!(
11790                Instant::now() < deadline,
11791                "module never entered the crash backoff"
11792            );
11793            sleep(Duration::from_millis(10)).await;
11794        }
11795
11796        let started = Instant::now();
11797        module.set_enabled(false).await.unwrap();
11798        let waited = started.elapsed();
11799
11800        assert!(
11801            waited < backoff / 2,
11802            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11803        );
11804        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11805
11806        // Outlast the backoff: the respawn it was counting down to must never run.
11807        sleep(backoff + Duration::from_millis(500)).await;
11808        let status = module.status().unwrap();
11809        assert_eq!(status.state, ModuleState::Disabled);
11810        assert_eq!(
11811            status.spawn_generation, 1,
11812            "module respawned after the operator disabled it"
11813        );
11814    }
11815
11816    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11817    /// the shape of nats-server, the program this rule exists for.
11818    #[cfg(unix)]
11819    fn protocol_none_sigterm_exits_clean_spec(
11820        module_id: &str,
11821        dir: &std::path::Path,
11822    ) -> (ModuleSpec, PathBuf, PathBuf) {
11823        let ready = dir.join("ready");
11824        let marker = dir.join("sigterm");
11825        let spec = ModuleSpec {
11826            module_id: module_id.to_string(),
11827            program: fake_aft_stub_path(),
11828            args: Vec::new(),
11829            env: vec![
11830                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11831                (
11832                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11833                    marker.display().to_string(),
11834                ),
11835                (
11836                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11837                    ready.display().to_string(),
11838                ),
11839            ],
11840            reserved: false,
11841            reserved_prefixes: Vec::new(),
11842            protocol: ModuleProtocol::None,
11843            overlap: Default::default(),
11844        };
11845        (spec, ready, marker)
11846    }
11847
11848    /// Wait for a file the child writes, so a signal is never sent before the
11849    /// child's SIGTERM handler is installed (the default disposition would
11850    /// kill it by signal and the exit would not be clean).
11851    #[cfg(unix)]
11852    async fn wait_for_file(path: &std::path::Path) {
11853        let deadline = Instant::now() + Duration::from_secs(10);
11854        while !path.exists() {
11855            assert!(
11856                Instant::now() < deadline,
11857                "{} never appeared",
11858                path.display()
11859            );
11860            sleep(Duration::from_millis(10)).await;
11861        }
11862    }
11863
11864    /// A protocol-none module that exits 0 because something OUTSIDE the
11865    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11866    /// the crash-path disposition rather than `stopped`.
11867    #[cfg(unix)]
11868    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11869    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11870        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11871        let (spec, ready, marker) =
11872            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11873        let supervisor = Supervisor::new_for_test(
11874            Arc::new(Registry::default()),
11875            RestartPolicy::new(3, Duration::ZERO),
11876        );
11877        let module = supervisor.spawn(spec).unwrap();
11878        wait_for_file(&ready).await;
11879        // The ready file proves the child installed its SIGTERM handler, not
11880        // that the supervisor has processed the privacy trampoline's exec
11881        // acknowledgement. On macOS status withholds the pid until then.
11882        let deadline = Instant::now() + Duration::from_secs(10);
11883        let first_pid = loop {
11884            let status = module.status().unwrap();
11885            if status.state == ModuleState::Running {
11886                if let Some(pid) = status.pid {
11887                    break pid;
11888                }
11889            }
11890            assert!(
11891                Instant::now() < deadline,
11892                "a running module must report its pid after exec confirmation: {status:?}"
11893            );
11894            sleep(Duration::from_millis(10)).await;
11895        };
11896
11897        rustix::process::kill_process(
11898            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11899            rustix::process::Signal::TERM,
11900        )
11901        .unwrap();
11902
11903        let deadline = Instant::now() + Duration::from_secs(10);
11904        let respawned = loop {
11905            let status = module.status().unwrap();
11906            if status.state == ModuleState::Running
11907                && status.pid.is_some_and(|pid| pid != first_pid)
11908            {
11909                break status;
11910            }
11911            assert!(
11912                Instant::now() < deadline,
11913                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11914            );
11915            sleep(Duration::from_millis(10)).await;
11916        };
11917        assert_eq!(respawned.spawn_generation, 2);
11918        assert!(
11919            marker.exists(),
11920            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11921        );
11922
11923        let history = module.terminal_history();
11924        assert_eq!(history.entries.len(), 1, "{history:?}");
11925        let entry = &history.entries[0];
11926        assert_eq!(entry.exit_code, Some(0));
11927        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11928        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11929
11930        module.stop().await.unwrap();
11931    }
11932
11933    /// Repeated unrequested clean exits of a protocol-none module spend the
11934    /// restart budget exactly as crashes do, and the module ends `failed` with
11935    /// the budget named.
11936    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11937    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11938        let supervisor = Supervisor::new_for_test(
11939            Arc::new(Registry::default()),
11940            RestartPolicy::new(1, Duration::ZERO),
11941        );
11942        let module = supervisor
11943            .spawn(ModuleSpec {
11944                module_id: "none-clean-exit-budget".to_string(),
11945                program: fake_aft_stub_path(),
11946                args: Vec::new(),
11947                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11948                reserved: false,
11949                reserved_prefixes: Vec::new(),
11950                protocol: ModuleProtocol::None,
11951                overlap: Default::default(),
11952            })
11953            .unwrap();
11954
11955        // Failed follows the terminal write; this deadline only bounds a hang,
11956        // not an assumed duration for the two launches or their exit recording.
11957        let deadline = Instant::now() + Duration::from_secs(10);
11958        loop {
11959            let status = module.status().unwrap();
11960            if status.state == ModuleState::Failed {
11961                break;
11962            }
11963            assert!(
11964                Instant::now() < deadline,
11965                "module never exhausted its budget: {status:?} {:?}",
11966                module.terminal_history()
11967            );
11968            sleep(Duration::from_millis(10)).await;
11969        }
11970        let history = module.terminal_history();
11971        assert_eq!(
11972            history
11973                .entries
11974                .iter()
11975                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11976                .collect::<Vec<_>>(),
11977            vec![
11978                (Some(0), TerminalDisposition::Restarting),
11979                (Some(0), TerminalDisposition::Failed),
11980            ]
11981        );
11982        let detail = history.entries[1]
11983            .disposition_detail
11984            .as_deref()
11985            .expect("a budget failure names the budget");
11986        assert!(detail.contains("max_restarts=1"), "{detail}");
11987        assert_eq!(module.status().unwrap().spawn_generation, 2);
11988    }
11989
11990    #[test]
11991    fn restart_budget_failure_is_published_after_its_terminal_record() {
11992        use std::io::{Read, Write};
11993        let supervisor = Supervisor::new_for_test(
11994            Arc::new(Registry::default()),
11995            RestartPolicy::new(0, Duration::ZERO),
11996        );
11997        let runtime = supervisor.runtime_config();
11998        let spec = ModuleSpec {
11999            protocol: ModuleProtocol::None,
12000            ..windowed_crash_spec("budget-publication-order")
12001        };
12002        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12003            ModuleState::Running,
12004            true,
12005        )));
12006        lock_snapshot(&snapshot).unwrap().process_alive = true;
12007        let reaped_snapshot = snapshot.clone();
12008        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12009        let history = runtime.terminal_ring.clone();
12010        let reap = std::thread::spawn(move || {
12011            tokio::runtime::Builder::new_current_thread()
12012                .enable_all()
12013                .build()
12014                .unwrap()
12015                .block_on(on_child_exit(
12016                    &spec,
12017                    runtime.restart_policy,
12018                    &supervisor.registry,
12019                    &reaped_snapshot,
12020                    &runtime.terminal_ring,
12021                    &runtime.spawn_events,
12022                    &runtime.child_roster,
12023                    ExitReport {
12024                        kind: ExitKind::Clean,
12025                        code: Some(0),
12026                        signal: None,
12027                        at_ms: 1,
12028                    },
12029                ))
12030        });
12031        let mut reached = [0];
12032        barrier.read_exact(&mut reached).unwrap();
12033        assert_eq!(&reached, b"R");
12034        let (before_state, before_alive) = {
12035            let state = lock_snapshot(&snapshot).unwrap();
12036            (state.state, state.process_alive)
12037        };
12038        let before_history = history.lock().unwrap().snapshot();
12039        barrier.write_all(b"X").unwrap();
12040        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12041        assert!(before_history.entries.is_empty());
12042        assert_ne!(
12043            before_state,
12044            ModuleState::Failed,
12045            "Failed was visible before its terminal record could be written"
12046        );
12047        assert!(before_alive);
12048        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12049        let history = history.lock().unwrap().snapshot();
12050        assert_eq!(history.entries.len(), 1);
12051        assert_eq!(history.entries[0].exit_code, Some(0));
12052        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12053    }
12054
12055    #[test]
12056    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12057        use std::io::{Read, Write};
12058
12059        let supervisor = Supervisor::new_for_test(
12060            Arc::new(Registry::default()),
12061            RestartPolicy::new(3, Duration::ZERO),
12062        );
12063        let mut runtime = supervisor.runtime_config();
12064        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12065        let snapshot = module.inner.snapshot.clone();
12066        lock_snapshot(&snapshot).unwrap().process_alive = true;
12067        runtime.terminal_ring = module.inner.terminal_ring.clone();
12068        let spec = windowed_crash_spec(module.module_id());
12069        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12070        let reaped_snapshot = snapshot.clone();
12071        let reap = std::thread::spawn(move || {
12072            tokio::runtime::Builder::new_current_thread()
12073                .enable_all()
12074                .build()
12075                .unwrap()
12076                .block_on(on_child_exit(
12077                    &spec,
12078                    runtime.restart_policy,
12079                    &supervisor.registry,
12080                    &reaped_snapshot,
12081                    &runtime.terminal_ring,
12082                    &runtime.spawn_events,
12083                    &runtime.child_roster,
12084                    ExitReport {
12085                        kind: ExitKind::Crash,
12086                        code: Some(2),
12087                        signal: None,
12088                        at_ms: 1,
12089                    },
12090                ))
12091        });
12092
12093        // Hold the crash handler between its budget decision and publication.
12094        // Connection cleanup must still see the recovery that this crash chose,
12095        // even when that recovery consumes the last available restart slot.
12096        let mut reached = [0];
12097        barrier.read_exact(&mut reached).unwrap();
12098        assert_eq!(&reached, b"R");
12099        let before_status = module.status().unwrap();
12100        let before_history = module.terminal_history();
12101        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12102        barrier.write_all(b"X").unwrap();
12103        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12104
12105        assert_eq!(before_status.state, ModuleState::Running);
12106        assert!(before_status.process_alive);
12107        assert_eq!(before_status.restart_count, 3);
12108        assert!(before_history.entries.is_empty());
12109        assert!(
12110            before_recovers,
12111            "connection cleanup would report a terminal close for a scheduled crash restart"
12112        );
12113        let restarting = module.status().unwrap();
12114        assert_eq!(restarting.state, ModuleState::Restarting);
12115        assert_eq!(restarting.restart_count, 3);
12116        assert!(module.will_recover_after_connection_loss().unwrap());
12117        let history = module.terminal_history();
12118        assert_eq!(history.entries.len(), 1);
12119        assert_eq!(
12120            history.entries[0].disposition,
12121            TerminalDisposition::Restarting
12122        );
12123
12124        // Once the replacement is running, its next crash must be judged
12125        // against the exhausted budget rather than the previous recovery.
12126        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12127            state.state = ModuleState::Running;
12128            state.respawn_pending = false;
12129        })
12130        .unwrap();
12131        assert!(!module.will_recover_after_connection_loss().unwrap());
12132        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12133            state.enabled = false;
12134            state.respawn_pending = true;
12135        })
12136        .unwrap();
12137        assert!(!module.will_recover_after_connection_loss().unwrap());
12138    }
12139
12140    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12141        use std::net::{TcpListener, TcpStream};
12142        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12143        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12144        let (barrier, _) = listener.accept().unwrap();
12145        for stream in [&writer, &barrier] {
12146            stream
12147                .set_read_timeout(Some(Duration::from_secs(10)))
12148                .unwrap();
12149            stream
12150                .set_write_timeout(Some(Duration::from_secs(10)))
12151                .unwrap();
12152        }
12153        *events.1.lock().unwrap() = Some(writer);
12154        barrier
12155    }
12156
12157    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12158        use std::io::{Read, Write};
12159        let barrier = events.1.lock().unwrap().take();
12160        if let Some(mut barrier) = barrier {
12161            barrier.write_all(b"R").unwrap();
12162            let mut release = [0];
12163            barrier.read_exact(&mut release).unwrap();
12164            assert_eq!(&release, b"X");
12165        }
12166    }
12167
12168    #[test]
12169    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12170        use std::io::{Read, Write};
12171
12172        let supervisor = Supervisor::new_for_test(
12173            Arc::new(Registry::default()),
12174            RestartPolicy::new(0, Duration::ZERO),
12175        );
12176        let mut runtime = supervisor.runtime_config();
12177        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12178        let snapshot = module.inner.snapshot.clone();
12179        runtime.terminal_ring = module.inner.terminal_ring.clone();
12180        let spec = ModuleSpec {
12181            module_id: module.module_id().into(),
12182            program: fake_aft_stub_path(),
12183            protocol: ModuleProtocol::None,
12184            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12185            ..windowed_crash_spec(module.module_id())
12186        };
12187        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12188        let reap = std::thread::spawn(move || {
12189            tokio::runtime::Builder::new_current_thread()
12190                .enable_all()
12191                .build()
12192                .unwrap()
12193                .block_on(async {
12194                    let mut child =
12195                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12196                    health_restart_child(
12197                        &spec,
12198                        &runtime,
12199                        &supervisor.registry,
12200                        &supervisor.process_liveness,
12201                        &snapshot,
12202                        &mut child,
12203                        SupervisorHealthStatus::Failing,
12204                        Some("health probe did not answer"),
12205                        1,
12206                    )
12207                    .await
12208                    .unwrap();
12209                    assert!(child.is_none());
12210                });
12211        });
12212        // The loopback barrier acknowledges entry into terminal recording by
12213        // health_restart_child's drain, then waits for the test's release byte.
12214        // Status and history are observed while that writer is held, not after
12215        // an elapsed-time guess about when the health action has completed.
12216        let mut reached = [0];
12217        barrier.read_exact(&mut reached).unwrap();
12218        assert_eq!(&reached, b"R");
12219        let before_status = module.status().unwrap();
12220        let before_history = module.terminal_history();
12221        barrier.write_all(b"X").unwrap();
12222        reap.join().unwrap();
12223        assert!(before_history.entries.is_empty());
12224        assert_ne!(
12225            before_status.state,
12226            ModuleState::Failed,
12227            "Failed was visible before its terminal record could be written"
12228        );
12229        assert!(
12230            before_status.process_alive,
12231            "process death was visible before its terminal record could be written"
12232        );
12233        let failed = module.status().unwrap();
12234        assert_eq!(failed.state, ModuleState::Failed);
12235        assert!(!failed.process_alive);
12236        let history = module.terminal_history();
12237        assert_eq!(history.entries.len(), 1);
12238        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12239        assert_eq!(
12240            history.entries[0].disposition_detail.as_deref(),
12241            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12242        );
12243    }
12244
12245    #[test]
12246    fn observed_exits_publish_process_death_after_terminal_history() {
12247        use std::io::{Read, Write};
12248        for (kind, enabled, shutdown, wait_error, expected_state) in [
12249            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12250            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12251            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12252            (
12253                ExitKind::DeliberateSeverance,
12254                true,
12255                false,
12256                false,
12257                ModuleState::Restarting,
12258            ),
12259            #[cfg(unix)]
12260            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12261            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12262        ] {
12263            let supervisor = Supervisor::new_for_test(
12264                Arc::new(Registry::default()),
12265                RestartPolicy::new(1, Duration::ZERO),
12266            );
12267            let runtime = supervisor.runtime_config();
12268            #[cfg(unix)]
12269            if shutdown {
12270                runtime.child_roster.close();
12271            }
12272            #[cfg(not(unix))]
12273            let _ = shutdown;
12274            let spec = windowed_crash_spec("exit-publication-order");
12275            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12276                ModuleState::Running,
12277                enabled,
12278            )));
12279            lock_snapshot(&snapshot).unwrap().process_alive = true;
12280            let reaped_snapshot = snapshot.clone();
12281            let history = runtime.terminal_ring.clone();
12282            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12283            let reap = std::thread::spawn(move || {
12284                if wait_error {
12285                    super::fail_after_wait_error(
12286                        &spec.module_id,
12287                        &reaped_snapshot,
12288                        &runtime.terminal_ring,
12289                        &runtime.spawn_events,
12290                        "wait failed".into(),
12291                    );
12292                } else {
12293                    tokio::runtime::Builder::new_current_thread()
12294                        .enable_all()
12295                        .build()
12296                        .unwrap()
12297                        .block_on(on_child_exit(
12298                            &spec,
12299                            runtime.restart_policy,
12300                            &supervisor.registry,
12301                            &reaped_snapshot,
12302                            &runtime.terminal_ring,
12303                            &runtime.spawn_events,
12304                            &runtime.child_roster,
12305                            ExitReport {
12306                                kind,
12307                                code: Some(0),
12308                                signal: None,
12309                                at_ms: 1,
12310                            },
12311                        ));
12312                }
12313            });
12314            let mut reached = [0];
12315            barrier.read_exact(&mut reached).unwrap();
12316            assert_eq!(&reached, b"R");
12317            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12318            let before_history = history.lock().unwrap().snapshot();
12319            barrier.write_all(b"X").unwrap();
12320            reap.join().unwrap();
12321            assert!(
12322                before_alive,
12323                "{kind:?}: death was published before recording"
12324            );
12325            assert!(before_history.entries.is_empty());
12326            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12327            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12328            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12329        }
12330    }
12331
12332    #[tokio::test]
12333    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12334        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12335        let path = dir.join("terminals.jsonl");
12336        std::fs::create_dir(&path).unwrap();
12337        let supervisor = Supervisor::new_for_test(
12338            Arc::new(Registry::default()),
12339            RestartPolicy::new(0, Duration::ZERO),
12340        )
12341        .with_terminal_journal(path, "health-budget-journal-failure".into());
12342        let runtime = supervisor.runtime_config();
12343        let spec = ModuleSpec {
12344            program: fake_aft_stub_path(),
12345            protocol: ModuleProtocol::None,
12346            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12347            ..windowed_crash_spec("health-budget-journal-failure")
12348        };
12349        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12350        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12351        health_restart_child(
12352            &spec,
12353            &runtime,
12354            &supervisor.registry,
12355            &supervisor.process_liveness,
12356            &snapshot,
12357            &mut child,
12358            SupervisorHealthStatus::Failing,
12359            None,
12360            1,
12361        )
12362        .await
12363        .unwrap();
12364        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12365        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12366        let history = runtime
12367            .terminal_ring
12368            .lock()
12369            .unwrap()
12370            .durable_history(&spec.module_id);
12371        assert!(history.journal_write_failures > 0);
12372        assert_eq!(history.entries.len(), 1);
12373        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12374    }
12375
12376    #[test]
12377    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12378        let supervisor = Supervisor::new_for_test(
12379            Arc::new(Registry::default()),
12380            RestartPolicy::new(0, Duration::ZERO),
12381        );
12382        let runtime = supervisor.runtime_config();
12383        let spec = ModuleSpec {
12384            program: fake_aft_stub_path(),
12385            protocol: ModuleProtocol::None,
12386            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12387            ..windowed_crash_spec("health-budget-recording-panic")
12388        };
12389        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12390        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12391            tokio::runtime::Builder::new_current_thread()
12392                .enable_all()
12393                .build()
12394                .unwrap()
12395                .block_on(async {
12396                    let mut child =
12397                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12398                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12399                    health_restart_child(
12400                        &spec,
12401                        &runtime,
12402                        &supervisor.registry,
12403                        &supervisor.process_liveness,
12404                        &snapshot,
12405                        &mut child,
12406                        SupervisorHealthStatus::Failing,
12407                        None,
12408                        1,
12409                    )
12410                    .await
12411                    .unwrap();
12412                });
12413        }));
12414        let panic = result.expect_err("recording must still unwind");
12415        assert_eq!(
12416            panic.downcast_ref::<String>().map(String::as_str),
12417            Some("spawn event sequence exhausted")
12418        );
12419        let state = lock_snapshot(&snapshot).unwrap();
12420        assert_eq!(state.state, ModuleState::Failed);
12421        assert!(!state.process_alive);
12422        assert!(state.last_exit.is_some());
12423    }
12424
12425    #[tokio::test]
12426    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12427        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12428        let path = dir.join("terminals.jsonl");
12429        std::fs::create_dir(&path).unwrap();
12430        let supervisor = Supervisor::new_for_test(
12431            Arc::new(Registry::default()),
12432            RestartPolicy::new(0, Duration::ZERO),
12433        )
12434        .with_terminal_journal(path, "budget-journal-failure".into());
12435        let runtime = supervisor.runtime_config();
12436        let spec = windowed_crash_spec("budget-journal-failure");
12437        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12438        assert!(matches!(
12439            on_child_exit(
12440                &spec,
12441                runtime.restart_policy,
12442                &supervisor.registry,
12443                &snapshot,
12444                &runtime.terminal_ring,
12445                &runtime.spawn_events,
12446                &runtime.child_roster,
12447                crash_exit_report(1),
12448            )
12449            .await,
12450            NextAction::Stop { .. }
12451        ));
12452        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12453        let history = runtime
12454            .terminal_ring
12455            .lock()
12456            .unwrap()
12457            .durable_history(&spec.module_id);
12458        assert!(history.journal_write_failures > 0);
12459        assert_eq!(history.entries.len(), 1);
12460        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12461    }
12462
12463    #[test]
12464    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12465        let supervisor = Supervisor::new_for_test(
12466            Arc::new(Registry::default()),
12467            RestartPolicy::new(0, Duration::ZERO),
12468        );
12469        let runtime = supervisor.runtime_config();
12470        let spec = windowed_crash_spec("budget-recording-panic");
12471        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12472        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12473        // Exhausting the event sequence makes emit_exited panic before the
12474        // terminal write, exercising publication on the recording unwind.
12475        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12476        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12477            tokio::runtime::Builder::new_current_thread()
12478                .enable_all()
12479                .build()
12480                .unwrap()
12481                .block_on(on_child_exit(
12482                    &spec,
12483                    runtime.restart_policy,
12484                    &supervisor.registry,
12485                    &snapshot,
12486                    &runtime.terminal_ring,
12487                    &runtime.spawn_events,
12488                    &runtime.child_roster,
12489                    crash_exit_report(1),
12490                ))
12491        }));
12492        let panic = result.err().expect("recording must still unwind");
12493        assert_eq!(
12494            panic.downcast_ref::<String>().map(String::as_str),
12495            Some("spawn event sequence exhausted")
12496        );
12497        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12498    }
12499
12500    /// A stop the supervisor itself requests still stops a protocol-none
12501    /// module, even though the child answers the SIGTERM with exit 0.
12502    #[cfg(unix)]
12503    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12504    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12505        for disable in [false, true] {
12506            let label = if disable {
12507                "none-requested-disable"
12508            } else {
12509                "none-requested-stop"
12510            };
12511            let dir = cortexkit_test_support::ScratchDir::new(label);
12512            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12513            let supervisor = Supervisor::new_for_test(
12514                Arc::new(Registry::default()),
12515                RestartPolicy::new(3, Duration::ZERO),
12516            );
12517            let module = supervisor.spawn(spec).unwrap();
12518            wait_for_file(&ready).await;
12519
12520            if disable {
12521                module.set_enabled(false).await.unwrap();
12522            } else {
12523                module.stop().await.unwrap();
12524            }
12525            assert!(
12526                marker.exists(),
12527                "{label}: the child must have left through its SIGTERM handler with exit 0"
12528            );
12529
12530            // Long enough for a zero-backoff respawn to have happened if the
12531            // exit had been treated as a crash.
12532            sleep(Duration::from_millis(500)).await;
12533            let status = module.status().unwrap();
12534            let expected = if disable {
12535                ModuleState::Disabled
12536            } else {
12537                ModuleState::Stopped
12538            };
12539            assert_eq!(status.state, expected, "{label}");
12540            assert_eq!(
12541                status.spawn_generation, 1,
12542                "{label}: respawned after a requested stop"
12543            );
12544            let history = module.terminal_history();
12545            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12546            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12547            assert_ne!(
12548                history.entries[0].disposition,
12549                TerminalDisposition::Restarting,
12550                "{label}"
12551            );
12552        }
12553    }
12554
12555    /// A subc-wire module that exits 0 on its own is still a stop: the
12556    /// protocol-none rule must not reach it.
12557    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12558    async fn subc_wire_clean_exit_is_still_a_stop() {
12559        let supervisor = Supervisor::new_for_test(
12560            Arc::new(Registry::default()),
12561            RestartPolicy::new(3, Duration::ZERO),
12562        );
12563        let module = supervisor
12564            .spawn(ModuleSpec {
12565                module_id: "wire-clean-exit".to_string(),
12566                program: fake_aft_stub_path(),
12567                args: Vec::new(),
12568                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12569                reserved: false,
12570                reserved_prefixes: Vec::new(),
12571                protocol: ModuleProtocol::Subc,
12572                overlap: Default::default(),
12573            })
12574            .unwrap();
12575
12576        let deadline = Instant::now() + Duration::from_secs(10);
12577        while module.terminal_history().entries.is_empty() {
12578            assert!(Instant::now() < deadline, "module never exited");
12579            sleep(Duration::from_millis(10)).await;
12580        }
12581        // Long enough for a zero-backoff respawn to have happened.
12582        sleep(Duration::from_millis(500)).await;
12583        let status = module.status().unwrap();
12584        assert_eq!(status.state, ModuleState::Stopped);
12585        assert_eq!(status.spawn_generation, 1);
12586        let history = module.terminal_history();
12587        assert_eq!(history.entries.len(), 1, "{history:?}");
12588        assert_eq!(history.entries[0].exit_code, Some(0));
12589        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12590    }
12591
12592    #[cfg(unix)]
12593    #[tokio::test]
12594    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12595        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12596        let record = dir.join("live-children.json");
12597        let supervisor = Supervisor::new_for_test(
12598            Arc::new(Registry::default()),
12599            RestartPolicy::new(0, Duration::ZERO),
12600        );
12601        let mut runtime = supervisor.runtime_config();
12602        runtime.child_roster.record_to(record.clone());
12603        let gate = Arc::new(super::ReloadExitRecordGate::default());
12604        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12605        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12606        let spec = ModuleSpec {
12607            module_id: "reload-exit-roster".into(),
12608            program: fake_aft_stub_path(),
12609            args: Vec::new(),
12610            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12611            reserved: false,
12612            reserved_prefixes: Vec::new(),
12613            protocol: ModuleProtocol::Subc,
12614            overlap: Default::default(),
12615        };
12616        let mut child = None;
12617        let reload = super::finish_reload_child(
12618            &spec,
12619            &runtime,
12620            &supervisor.registry,
12621            &supervisor.process_liveness,
12622            &snapshot,
12623            &mut child,
12624        );
12625        tokio::pin!(reload);
12626        tokio::select! {
12627            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12628            _ = gate.reached.notified() => {}
12629        }
12630        assert!(runtime
12631            .terminal_ring
12632            .lock()
12633            .unwrap()
12634            .snapshot()
12635            .entries
12636            .is_empty());
12637        assert_eq!(
12638            crate::live_children::read_record(&record).unwrap().len(),
12639            1,
12640            "shutdown must still wait for the reaped child until its terminal record exists"
12641        );
12642        runtime.child_roster.close();
12643        gate.resume.notify_one();
12644        assert!(reload.await.is_err());
12645        assert!(crate::live_children::read_record(&record)
12646            .unwrap()
12647            .is_empty());
12648        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12649        assert_eq!(history.entries.len(), 1);
12650        assert_eq!(
12651            history.entries[0].disposition,
12652            TerminalDisposition::DaemonShutdown
12653        );
12654    }
12655
12656    /// Each restart-producing arm has its own state transition. Keeping their
12657    /// lifetime count assertions adjacent prevents a later new arm from silently
12658    /// spending budget without recording the historical restart.
12659    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12660    async fn every_restart_increment_path_advances_lifetime_count() {
12661        let supervisor = Supervisor::new_for_test(
12662            Arc::new(Registry::default()),
12663            RestartPolicy::new(1, Duration::ZERO),
12664        );
12665        let runtime = supervisor.runtime_config();
12666        let spec = ModuleSpec {
12667            module_id: "lifetime-increment-path".to_string(),
12668            program: PathBuf::from("/unused/lifetime-increment-path"),
12669            args: Vec::new(),
12670            env: Vec::new(),
12671            reserved: false,
12672            reserved_prefixes: Vec::new(),
12673            protocol: ModuleProtocol::Subc,
12674            overlap: Default::default(),
12675        };
12676
12677        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12678        assert!(matches!(
12679            on_child_exit(
12680                &spec,
12681                runtime.restart_policy,
12682                &supervisor.registry,
12683                &crash_snapshot,
12684                &runtime.terminal_ring,
12685                &runtime.spawn_events,
12686                &runtime.child_roster,
12687                ExitReport {
12688                    kind: ExitKind::Crash,
12689                    code: Some(1),
12690                    signal: None,
12691                    at_ms: 1,
12692                },
12693            )
12694            .await,
12695            NextAction::Restart { schedule: _ }
12696        ));
12697        let (crash_restarts, crash_lifetime) = {
12698            let state = lock_snapshot(&crash_snapshot).unwrap();
12699            (state.crash_restarts.len(), state.lifetime_restarts)
12700        };
12701        assert_eq!(crash_restarts, 1);
12702        assert_eq!(crash_lifetime, 1);
12703
12704        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12705        let mut health_child = None;
12706        assert!(matches!(
12707            health_restart_child(
12708                &spec,
12709                &runtime,
12710                &supervisor.registry,
12711                &supervisor.process_liveness,
12712                &health_snapshot,
12713                &mut health_child,
12714                SupervisorHealthStatus::Failing,
12715                None,
12716                2,
12717            )
12718            .await,
12719            Ok(())
12720        ));
12721        assert!(health_child.is_none());
12722        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12723        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12724        let (health_restarts, health_lifetime) = {
12725            let state = lock_snapshot(&health_snapshot).unwrap();
12726            (state.crash_restarts.len(), state.lifetime_restarts)
12727        };
12728        assert_eq!(health_restarts, 1);
12729        assert_eq!(health_lifetime, 1);
12730
12731        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12732        let mut reload_child = None;
12733        assert!(matches!(
12734            handle_reload_spawn_failure(
12735                &spec,
12736                &runtime,
12737                &supervisor.process_liveness,
12738                &reload_snapshot,
12739                &mut reload_child,
12740                "forced reload spawn failure".to_string(),
12741            )
12742            .await,
12743            Err(SuperviseError::ReloadFailed { .. })
12744        ));
12745        let (reload_restarts, reload_lifetime) = {
12746            let state = lock_snapshot(&reload_snapshot).unwrap();
12747            (state.crash_restarts.len(), state.lifetime_restarts)
12748        };
12749        assert_eq!(reload_restarts, 1);
12750        assert_eq!(reload_lifetime, 1);
12751    }
12752
12753    #[tokio::test]
12754    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12755        let supervisor = Supervisor::new_for_test(
12756            Arc::new(Registry::default()),
12757            RestartPolicy::new(3, Duration::ZERO),
12758        );
12759        let runtime = supervisor.runtime_config();
12760        let spec = ModuleSpec {
12761            module_id: "deliberately-severed".to_string(),
12762            program: PathBuf::from("/unused/deliberately-severed"),
12763            args: Vec::new(),
12764            env: Vec::new(),
12765            reserved: false,
12766            reserved_prefixes: Vec::new(),
12767            protocol: ModuleProtocol::Subc,
12768            overlap: Default::default(),
12769        };
12770        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12771        let process = ProcessIdentity {
12772            pid: 41,
12773            start_time: 101,
12774        };
12775        record_deliberate_severance(&snapshot, process).unwrap();
12776        let exit_report = apply_deliberate_severance_marker(
12777            &snapshot,
12778            Some(process),
12779            ExitReport {
12780                kind: ExitKind::Crash,
12781                code: Some(1),
12782                signal: None,
12783                at_ms: 1,
12784            },
12785        );
12786        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12787
12788        assert!(matches!(
12789            on_child_exit(
12790                &spec,
12791                runtime.restart_policy,
12792                &supervisor.registry,
12793                &snapshot,
12794                &runtime.terminal_ring,
12795                &runtime.spawn_events,
12796                &runtime.child_roster,
12797                exit_report,
12798            )
12799            .await,
12800            NextAction::Restart { schedule: _ }
12801        ));
12802        let state = lock_snapshot(&snapshot).unwrap();
12803        assert_eq!(state.lifetime_restarts, 1);
12804        assert_eq!(state.crash_restarts.len(), 0);
12805    }
12806
12807    #[tokio::test]
12808    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12809        let supervisor = Supervisor::new_for_test(
12810            Arc::new(Registry::default()),
12811            RestartPolicy::new(3, Duration::ZERO),
12812        );
12813        let runtime = supervisor.runtime_config();
12814        let spec = ModuleSpec {
12815            module_id: "genuine-crash".to_string(),
12816            program: PathBuf::from("/unused/genuine-crash"),
12817            args: Vec::new(),
12818            env: Vec::new(),
12819            reserved: false,
12820            reserved_prefixes: Vec::new(),
12821            protocol: ModuleProtocol::Subc,
12822            overlap: Default::default(),
12823        };
12824        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12825
12826        assert!(matches!(
12827            on_child_exit(
12828                &spec,
12829                runtime.restart_policy,
12830                &supervisor.registry,
12831                &snapshot,
12832                &runtime.terminal_ring,
12833                &runtime.spawn_events,
12834                &runtime.child_roster,
12835                ExitReport {
12836                    kind: ExitKind::Crash,
12837                    code: Some(1),
12838                    signal: None,
12839                    at_ms: 1,
12840                },
12841            )
12842            .await,
12843            NextAction::Restart { schedule: _ }
12844        ));
12845        let state = lock_snapshot(&snapshot).unwrap();
12846        assert_eq!(state.lifetime_restarts, 1);
12847        assert_eq!(state.crash_restarts.len(), 1);
12848    }
12849
12850    fn crash_exit_report(at_ms: u64) -> ExitReport {
12851        ExitReport {
12852            kind: ExitKind::Crash,
12853            code: Some(1),
12854            signal: None,
12855            at_ms,
12856        }
12857    }
12858
12859    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12860        ModuleSpec {
12861            module_id: module_id.to_string(),
12862            program: PathBuf::from("/unused").join(module_id),
12863            args: Vec::new(),
12864            env: Vec::new(),
12865            reserved: false,
12866            reserved_prefixes: Vec::new(),
12867            protocol: ModuleProtocol::Subc,
12868            overlap: Default::default(),
12869        }
12870    }
12871
12872    /// A real crash loop still stops. Three crashes with nothing aging out spend
12873    /// a budget of two and the third respawn is refused, and both surfaces an
12874    /// operator has -- the log line and the retained terminal record -- name the
12875    /// window rather than only the cap, because `max_restarts=2` alone is what
12876    /// this budget used to mean.
12877    #[tokio::test]
12878    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12879        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12880        let supervisor = Supervisor::new_for_test(
12881            Arc::new(Registry::default()),
12882            RestartPolicy::new(2, Duration::ZERO),
12883        );
12884        let runtime = supervisor.runtime_config();
12885        let spec = windowed_crash_spec("crash-loop-in-window");
12886        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12887
12888        for attempt in 1..=2 {
12889            assert!(
12890                matches!(
12891                    on_child_exit(
12892                        &spec,
12893                        runtime.restart_policy,
12894                        &supervisor.registry,
12895                        &snapshot,
12896                        &runtime.terminal_ring,
12897                        &runtime.spawn_events,
12898                        &runtime.child_roster,
12899                        crash_exit_report(attempt),
12900                    )
12901                    .await,
12902                    NextAction::Restart { schedule: _ }
12903                ),
12904                "crash {attempt} is inside the budget and must respawn"
12905            );
12906        }
12907
12908        assert!(matches!(
12909            on_child_exit(
12910                &spec,
12911                runtime.restart_policy,
12912                &supervisor.registry,
12913                &snapshot,
12914                &runtime.terminal_ring,
12915                &runtime.spawn_events,
12916                &runtime.child_roster,
12917                crash_exit_report(3),
12918            )
12919            .await,
12920            NextAction::Stop { .. }
12921        ));
12922
12923        {
12924            let state = lock_snapshot(&snapshot).unwrap();
12925            assert_eq!(state.state, ModuleState::Failed);
12926            assert_eq!(state.crash_restarts.len(), 2);
12927            assert_eq!(state.lifetime_restarts, 2);
12928        }
12929
12930        let history = runtime
12931            .terminal_ring
12932            .lock()
12933            .expect("terminal ring is not poisoned")
12934            .snapshot();
12935        let last = history
12936            .entries
12937            .last()
12938            .expect("the refused crash is retained");
12939        assert_eq!(last.disposition, TerminalDisposition::Failed);
12940        assert_eq!(
12941            last.disposition_detail.as_deref(),
12942            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12943        );
12944
12945        let captured = crate::router::test_log::captured_logs(&logs);
12946        assert!(
12947            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12948            "the stop must be logged with its window: {captured}"
12949        );
12950    }
12951
12952    /// The rate, stated as a test: three crashes where the first has aged past
12953    /// the window are two crashes as far as the budget is concerned, so the
12954    /// third respawn is allowed and the ring holds only the two recent ones.
12955    ///
12956    /// This is the case a lifetime counter got wrong -- and the case the daemon
12957    /// now hits routinely, since a module exits non-zero every time its
12958    /// connection to the daemon drops.
12959    #[tokio::test]
12960    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12961        let supervisor = Supervisor::new_for_test(
12962            Arc::new(Registry::default()),
12963            RestartPolicy::new(2, Duration::ZERO),
12964        );
12965        let runtime = supervisor.runtime_config();
12966        let spec = windowed_crash_spec("crash-across-windows");
12967        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12968
12969        for attempt in 1..=2 {
12970            assert!(matches!(
12971                on_child_exit(
12972                    &spec,
12973                    runtime.restart_policy,
12974                    &supervisor.registry,
12975                    &snapshot,
12976                    &runtime.terminal_ring,
12977                    &runtime.spawn_events,
12978                    &runtime.child_roster,
12979                    crash_exit_report(attempt),
12980                )
12981                .await,
12982                NextAction::Restart { schedule: _ }
12983            ));
12984        }
12985
12986        // The oldest crash moves out of the window; nothing else about the
12987        // module changes.
12988        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12989            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12990        })
12991        .unwrap();
12992
12993        assert!(
12994            matches!(
12995                on_child_exit(
12996                    &spec,
12997                    runtime.restart_policy,
12998                    &supervisor.registry,
12999                    &snapshot,
13000                    &runtime.terminal_ring,
13001                    &runtime.spawn_events,
13002                    &runtime.child_roster,
13003                    crash_exit_report(3),
13004                )
13005                .await,
13006                NextAction::Restart { schedule: _ }
13007            ),
13008            "a crash older than the window must not hold a budget slot"
13009        );
13010
13011        let state = lock_snapshot(&snapshot).unwrap();
13012        assert_eq!(state.state, ModuleState::Restarting);
13013        assert_eq!(
13014            state.crash_restarts.len(),
13015            2,
13016            "the aged instant is dropped and the new one takes its place"
13017        );
13018        assert_eq!(
13019            state.lifetime_restarts, 3,
13020            "the ledger counts every restart, including the ones the window forgot"
13021        );
13022    }
13023
13024    /// An operator restart hands the budget back whole, and the ledger keeps
13025    /// counting. Those are different questions -- "how close is this module to
13026    /// being stopped" and "how many times has it been replaced" -- and the
13027    /// operator action answers only the first.
13028    #[tokio::test]
13029    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13030        let supervisor = Supervisor::new_for_test(
13031            Arc::new(Registry::default()),
13032            RestartPolicy::new(2, Duration::ZERO),
13033        );
13034        let runtime = supervisor.runtime_config();
13035        let spec = windowed_crash_spec("operator-cleared-budget");
13036        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13037
13038        for attempt in 1..=2 {
13039            assert!(matches!(
13040                on_child_exit(
13041                    &spec,
13042                    runtime.restart_policy,
13043                    &supervisor.registry,
13044                    &snapshot,
13045                    &runtime.terminal_ring,
13046                    &runtime.spawn_events,
13047                    &runtime.child_roster,
13048                    crash_exit_report(attempt),
13049                )
13050                .await,
13051                NextAction::Restart { schedule: _ }
13052            ));
13053        }
13054
13055        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13056        {
13057            let state = lock_snapshot(&snapshot).unwrap();
13058            assert!(
13059                state.crash_restarts.is_empty(),
13060                "an operator restart returns the full budget"
13061            );
13062            assert_eq!(
13063                state.lifetime_restarts, 2,
13064                "clearing the budget must not unmake the crashes"
13065            );
13066        }
13067
13068        assert!(
13069            matches!(
13070                on_child_exit(
13071                    &spec,
13072                    runtime.restart_policy,
13073                    &supervisor.registry,
13074                    &snapshot,
13075                    &runtime.terminal_ring,
13076                    &runtime.spawn_events,
13077                    &runtime.child_roster,
13078                    crash_exit_report(3),
13079                )
13080                .await,
13081                NextAction::Restart { schedule: _ }
13082            ),
13083            "the cleared budget must be spendable again"
13084        );
13085        let state = lock_snapshot(&snapshot).unwrap();
13086        assert_eq!(state.crash_restarts.len(), 1);
13087        assert_eq!(state.lifetime_restarts, 3);
13088    }
13089
13090    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13091    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13092        let severed = ProcessIdentity {
13093            pid: 41,
13094            start_time: 101,
13095        };
13096        let successor = ProcessIdentity {
13097            pid: 41,
13098            start_time: 202,
13099        };
13100        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13101        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13102            state.pid = Some(successor.pid);
13103            state.process_start_time = Some(successor.start_time);
13104        })
13105        .unwrap();
13106        assert!(!module.record_deliberate_severance(severed).unwrap());
13107
13108        let exit_report = apply_deliberate_severance_marker(
13109            &module.inner.snapshot,
13110            Some(successor),
13111            ExitReport {
13112                kind: ExitKind::Crash,
13113                code: Some(1),
13114                signal: None,
13115                at_ms: 1,
13116            },
13117        );
13118
13119        assert_eq!(exit_report.kind, ExitKind::Crash);
13120    }
13121
13122    #[tokio::test]
13123    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13124        let registry = Registry::default();
13125        let supervisor = Supervisor::new_for_test(
13126            Arc::new(Registry::default()),
13127            RestartPolicy::new(3, Duration::ZERO),
13128        );
13129        let runtime = supervisor.runtime_config();
13130        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13131        let spec = ModuleSpec {
13132            module_id: "drain-deliberate-severance".to_string(),
13133            program: fake_aft_stub_path(),
13134            args: Vec::new(),
13135            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13136            reserved: false,
13137            reserved_prefixes: Vec::new(),
13138            protocol: ModuleProtocol::Subc,
13139            overlap: Default::default(),
13140        };
13141        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13142        let process = ProcessIdentity {
13143            pid: 41,
13144            start_time: 101,
13145        };
13146        child.process_identity = Some(process);
13147        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13148            state.pid = Some(process.pid);
13149            state.process_start_time = Some(process.start_time);
13150        })
13151        .unwrap();
13152        record_deliberate_severance(&snapshot, process).unwrap();
13153
13154        drain_child_to_state(
13155            &spec.module_id,
13156            spec.protocol,
13157            // The child exits on its own; no signal may change the exit this
13158            // test classifies.
13159            StopNotice::SentOverConnection,
13160            &registry,
13161            None,
13162            &snapshot,
13163            &runtime.terminal_ring,
13164            &runtime.spawn_events,
13165            child,
13166            Duration::from_secs(1),
13167            ModuleState::Stopped,
13168            Some(false),
13169        )
13170        .await
13171        .unwrap();
13172
13173        let state = lock_snapshot(&snapshot).unwrap();
13174        assert_eq!(
13175            state.last_exit.as_ref().map(|exit| exit.kind),
13176            Some(ExitKind::DeliberateSeverance)
13177        );
13178        assert_eq!(state.lifetime_restarts, 1);
13179        assert_eq!(state.crash_restarts.len(), 0);
13180        drop(state);
13181        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13182        assert_eq!(
13183            history.entries[0].exit_kind,
13184            subc_control::TerminalExitKind::DeliberateSeverance
13185        );
13186    }
13187
13188    #[tokio::test]
13189    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13190        let registry = Registry::default();
13191        let supervisor = Supervisor::new_for_test(
13192            Arc::new(Registry::default()),
13193            RestartPolicy::new(3, Duration::ZERO),
13194        );
13195        let runtime = supervisor.runtime_config();
13196        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13197        let spec = ModuleSpec {
13198            module_id: "ordinary-drain".to_string(),
13199            program: fake_aft_stub_path(),
13200            args: Vec::new(),
13201            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13202            reserved: false,
13203            reserved_prefixes: Vec::new(),
13204            protocol: ModuleProtocol::Subc,
13205            overlap: Default::default(),
13206        };
13207        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13208
13209        drain_child_to_state(
13210            &spec.module_id,
13211            spec.protocol,
13212            // The child exits on its own; no signal may change the exit this
13213            // test classifies.
13214            StopNotice::SentOverConnection,
13215            &registry,
13216            None,
13217            &snapshot,
13218            &runtime.terminal_ring,
13219            &runtime.spawn_events,
13220            child,
13221            Duration::from_secs(1),
13222            ModuleState::Stopped,
13223            Some(false),
13224        )
13225        .await
13226        .unwrap();
13227
13228        let state = lock_snapshot(&snapshot).unwrap();
13229        assert_eq!(
13230            state.last_exit.as_ref().map(|exit| exit.kind),
13231            Some(ExitKind::Crash)
13232        );
13233        assert_eq!(state.lifetime_restarts, 0);
13234        assert_eq!(state.crash_restarts.len(), 0);
13235    }
13236
13237    #[test]
13238    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13239        // The server's generic fatal-routing branch only knows that the
13240        // connection failed; it does not know that the daemon deliberately
13241        // initiated a process-killing severance. Keep this seam explicit so a
13242        // future connection error path cannot silently reintroduce the stale
13243        // exemption that mislabels a later genuine crash.
13244        assert!(!include_str!("server.rs")
13245            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13246    }
13247
13248    /// The `route.closed` `drained` value must be the quiescence wait's own
13249    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13250    /// measurement at all and `false` is the one honest constant. This is the exact
13251    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13252    /// on every return path, including the one that used to return early via `?`
13253    /// with `route.closing` already sent and no `route.closed` ever following.
13254    #[test]
13255    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13256        assert!(drained_after_quiescence_wait(&Ok(true)));
13257        assert!(!drained_after_quiescence_wait(&Ok(false)));
13258        assert!(!drained_after_quiescence_wait(&Err(
13259            SuperviseError::StatePoisoned { module_id: None }
13260        )));
13261    }
13262
13263    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13264    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13265    /// already reaped out-of-band) still leaves a terminal record rather than none
13266    /// at all. Triggering the real `wait()` I/O error from an integration test would
13267    /// need a genuine already-reaped-child race, which is OS-specific and not
13268    /// something this suite attempts elsewhere; this test instead verifies the
13269    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13270    /// the call site itself is verified by inspection to sit in that exact arm.
13271    #[test]
13272    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13273        let ring = Arc::new(Mutex::new(TerminalRing::new(
13274            TerminalRingConfig::default(),
13275            0,
13276        )));
13277        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13278
13279        let snapshot = ring.lock().unwrap().snapshot();
13280        assert_eq!(snapshot.entries.len(), 1);
13281        let entry = &snapshot.entries[0];
13282        assert_eq!(entry.exit_code, None);
13283        assert_eq!(entry.exit_signal, None);
13284        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13285    }
13286
13287    #[test]
13288    fn wait_error_exit_path_preserves_spawn_event_density() {
13289        let feed = super::SpawnEventFeed::default();
13290        feed.configure_incarnation("wait-error-density".to_string());
13291        feed.emit_spawned("wait-error", 41, 1);
13292        let ring = Arc::new(Mutex::new(TerminalRing::new(
13293            TerminalRingConfig::default(),
13294            0,
13295        )));
13296
13297        record_wait_error_terminal("wait-error", &ring, &feed);
13298        feed.emit_spawned("after-wait-error", 42, 2);
13299
13300        let state = feed.0.lock().unwrap();
13301        let sequences = state
13302            .events
13303            .iter()
13304            .map(|event| event.cursor.seq)
13305            .collect::<Vec<_>>();
13306        assert_eq!(sequences, vec![1, 2, 3]);
13307        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13308        assert_eq!(state.events[1].exit_code, None);
13309        assert_eq!(state.events[1].exit_signal, None);
13310    }
13311
13312    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13313    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13314    /// not a clean exit it never actually observed.
13315    #[test]
13316    fn wait_error_exit_report_is_classified_as_a_crash() {
13317        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13318    }
13319}
13320
13321#[cfg(all(test, unix))]
13322mod restart_ordering_tests {
13323    use super::*;
13324    use std::io::Write;
13325    use tokio::io::AsyncReadExt;
13326
13327    #[test]
13328    #[ignore = "child process fixture"]
13329    fn serving_child() {
13330        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13331            return;
13332        };
13333        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13334        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13335        for stream in listener.incoming() {
13336            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13337        }
13338    }
13339
13340    async fn restart_before_publication(requests: usize) {
13341        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13342        let address_file = home.path().join("address");
13343        let spec = ModuleSpec {
13344            module_id: "restart-serving-child".into(),
13345            program: std::env::current_exe().unwrap(),
13346            args: vec![
13347                "--exact".into(),
13348                "supervise::restart_ordering_tests::serving_child".into(),
13349                "--ignored".into(),
13350                "--nocapture".into(),
13351            ],
13352            env: vec![(
13353                "RESTART_TEST_ADDRESS_FILE".into(),
13354                address_file.display().to_string(),
13355            )],
13356            reserved: false,
13357            reserved_prefixes: vec![],
13358            protocol: ModuleProtocol::None,
13359            overlap: Default::default(),
13360        };
13361        let mut supervisor =
13362            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13363        let (spawned_tx, spawned_rx) = oneshot::channel();
13364        let spawned_tx = Mutex::new(Some(spawned_tx));
13365        let (release_tx, release_rx) = std::sync::mpsc::channel();
13366        let release_rx = Mutex::new(release_rx);
13367        // Hold publication, not the child: it must answer a real request before
13368        // the restart enters the queue, even though status still has generation 0.
13369        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13370            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13371            release_rx
13372                .lock()
13373                .unwrap()
13374                .recv_timeout(Duration::from_secs(20))
13375                .unwrap();
13376        })));
13377        let registered = supervisor
13378            .register_configured_with_health(
13379                spec.clone(),
13380                true,
13381                HealthConfig::default(),
13382                None,
13383                RestartPolicy::new(3, Duration::ZERO),
13384            )
13385            .unwrap();
13386        let module = registered.module.clone();
13387        let startup = tokio::task::spawn_blocking(move || {
13388            supervisor.start_registered_configured(registered, None)
13389        });
13390        spawned_rx.await.unwrap();
13391        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13392            let address = loop {
13393                if let Ok(address) = std::fs::read_to_string(&address_file) {
13394                    if !address.is_empty() {
13395                        break address;
13396                    }
13397                }
13398                tokio::time::sleep(Duration::from_millis(1)).await;
13399            };
13400            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13401            let mut reply = String::new();
13402            stream.read_to_string(&mut reply).await.unwrap();
13403            reply.trim().parse::<u32>().unwrap()
13404        })
13405        .await
13406        .expect("child must serve before generation publication");
13407        assert_eq!(
13408            lock_snapshot(&module.inner.snapshot)
13409                .unwrap()
13410                .spawn_generation,
13411            0
13412        );
13413        let mut replies = Vec::new();
13414        for _ in 0..requests {
13415            let mut restart = Box::pin(module.restart(Some(0)));
13416            // Poll the public API through queue admission, leaving only its ACK
13417            // pending. The monitor cannot consume the command until released.
13418            std::future::poll_fn(|cx| {
13419                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13420                std::task::Poll::Ready(())
13421            })
13422            .await;
13423            replies.push(restart);
13424        }
13425        release_tx.send(()).unwrap();
13426        startup.await.unwrap().unwrap();
13427        for reply in replies {
13428            reply.await.unwrap();
13429        }
13430        // Restart schedules its replacement after completing the handler. Wait
13431        // for that separate actor turn before placing the command barrier.
13432        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13433            loop {
13434                if lock_snapshot(&module.inner.snapshot)
13435                    .unwrap()
13436                    .spawn_generation
13437                    > 1
13438                {
13439                    break;
13440                }
13441                tokio::time::sleep(Duration::from_millis(1)).await;
13442            }
13443        })
13444        .await;
13445        // Commands are handled serially. The update-spec reply proves every
13446        // restart handler has finished before we inspect the snapshot; restart's
13447        // own ACK only acknowledges initiation, not replacement completion.
13448        module.update_spec_for_test(spec).await.unwrap();
13449        let (generation, pid) = {
13450            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13451            (state.spawn_generation, state.pid)
13452        };
13453        module.stop().await.unwrap();
13454        replacement.expect("restart must publish a replacement for the already-serving child");
13455        assert_ne!(
13456            pid,
13457            Some(old_pid),
13458            "a serving child predating the restart must be replaced"
13459        );
13460        assert_eq!(generation, 2, "queued restarts must share one replacement");
13461    }
13462
13463    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13464    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13465        restart_before_publication(1).await;
13466    }
13467
13468    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13469    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13470        restart_before_publication(2).await;
13471    }
13472
13473    #[tokio::test]
13474    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13475        let supervisor =
13476            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13477        let registered = supervisor
13478            .register_configured_with_health(
13479                ModuleSpec {
13480                    module_id: "queued-before-replacement".into(),
13481                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
13482                    // macOS a module whose running image is not the configured
13483                    // program is refused once its exec is confirmed (see
13484                    // `record_write_stall_tests::sleeping_spec`).
13485                    program: "/bin/sleep".into(),
13486                    args: vec!["600".into()],
13487                    env: vec![],
13488                    reserved: false,
13489                    reserved_prefixes: vec![],
13490                    protocol: ModuleProtocol::None,
13491                    overlap: Default::default(),
13492                },
13493                true,
13494                HealthConfig::default(),
13495                None,
13496                RestartPolicy::default(),
13497            )
13498            .unwrap();
13499        let RegisteredConfiguredModule {
13500            mut spec,
13501            mut runtime,
13502            module,
13503            mut commands,
13504        } = registered;
13505        let snapshot = &module.inner.snapshot;
13506        let mut restart = Box::pin(module.restart(Some(0)));
13507        std::future::poll_fn(|cx| {
13508            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13509            std::task::Poll::Ready(())
13510        })
13511        .await;
13512        // Drive the actor explicitly so a replacement is definitely spawned
13513        // after receipt, but before this queued command is dequeued.
13514        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13515        let replacement_pid = child.as_ref().unwrap().id();
13516        let command = commands.recv().await.unwrap();
13517        assert!(
13518            handle_supervisor_command(
13519                command,
13520                &mut spec,
13521                &mut runtime,
13522                &supervisor.registry,
13523                &supervisor.process_liveness,
13524                snapshot,
13525                &mut child,
13526                &mut commands,
13527                &mut VecDeque::new(),
13528            )
13529            .await
13530        );
13531        restart.await.unwrap();
13532        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13533        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13534        if let Some(mut child) = child {
13535            child.start_kill().unwrap();
13536            child.wait().await.unwrap();
13537        }
13538        assert_eq!(
13539            kept_pid, replacement_pid,
13540            "a newer replacement must survive"
13541        );
13542        assert!(
13543            !pending,
13544            "a satisfied restart must not schedule another spawn"
13545        );
13546    }
13547}
13548
13549#[cfg(all(test, unix))]
13550mod health_event_tests {
13551    use super::*;
13552    use cortexkit_test_support::ScratchDir;
13553    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13554
13555    struct Actor {
13556        module: SupervisedModule,
13557        registry: Arc<Registry>,
13558        forwarding: Arc<ForwardingTable>,
13559        spec: ModuleSpec,
13560        health: HealthConfig,
13561        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13562        _home: ScratchDir,
13563    }
13564
13565    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13566        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13567        // for an observed condition, rather than letting paused time auto-advance
13568        // deadlines or assuming a fixed number of yields finishes the work.
13569        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13570        let tick = Instant::now();
13571        loop {
13572            if let Some(value) = observe() {
13573                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13574                return value;
13575            }
13576            assert!(
13577                std::time::Instant::now() < deadline,
13578                "timed out waiting for {reason}"
13579            );
13580            tokio::task::yield_now().await;
13581        }
13582    }
13583
13584    impl Actor {
13585        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13586            let home = ScratchDir::new("health-events");
13587            let registry = Arc::new(Registry::default());
13588            let forwarding = Arc::new(ForwardingTable::default());
13589            let health = HealthConfig {
13590                cadence,
13591                deadline: Duration::from_secs(3600),
13592                ..HealthConfig::default()
13593            };
13594            let supervisor =
13595                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13596                    .with_forwarding(forwarding.clone())
13597                    .with_health_config(health.clone());
13598            let spec = ModuleSpec {
13599                module_id: "event-health-child".into(),
13600                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
13601                // the privacy trampoline refuses a module whose running image
13602                // is not the configured program, and `start` below waits for
13603                // the process to be confirmed (`reported_pid`). The shell form
13604                // passed only while the image check ran before the shell had
13605                // replaced itself with `sleep` (see
13606                // `record_write_stall_tests::sleeping_spec`).
13607                program: PathBuf::from("/bin/sleep"),
13608                // The process stays alive but has no real bus peer. Registrations
13609                // below exercise the same registry writes as accepted HELLOs.
13610                args: vec!["600".into()],
13611                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13612                    .into_iter()
13613                    .map(|key| (key.into(), home.path().display().to_string()))
13614                    .collect(),
13615                reserved: false,
13616                reserved_prefixes: vec![],
13617                protocol,
13618                overlap: Default::default(),
13619            };
13620            let module = supervisor.spawn(spec.clone()).unwrap();
13621            let (_, rx) = mpsc::channel(8);
13622            let actor = Self {
13623                module,
13624                registry,
13625                forwarding,
13626                spec,
13627                health,
13628                rx,
13629                _home: home,
13630            };
13631            actor
13632                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13633                .await;
13634            actor
13635        }
13636
13637        fn turns(&self) -> u64 {
13638            lock_snapshot(&self.module.inner.snapshot)
13639                .unwrap()
13640                .actor_turns
13641        }
13642
13643        async fn wait_for_select(
13644            &self,
13645            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13646        ) -> ActorSelectCheckpoint {
13647            wait_for("actor select checkpoint", || {
13648                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13649                state.actor_select.filter(|checkpoint| {
13650                    checkpoint.generation == state.spawn_generation
13651                        && state.state == ModuleState::Running
13652                        && state.process_alive
13653                        && state.reported_pid().is_some()
13654                        && expected(checkpoint)
13655                })
13656            })
13657            .await
13658        }
13659
13660        fn hello(&mut self, connection: u64, health: bool) {
13661            let manifest =
13662                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13663                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13664                    .build();
13665            let connection = ConnectionId::new(connection);
13666            let (tx, rx) = mpsc::channel(8);
13667            self.rx = rx;
13668            self.forwarding
13669                .register_module_connection(
13670                    connection,
13671                    self.spec.module_id.clone(),
13672                    subc_protocol::PROTOCOL_VERSION,
13673                    Concurrency::ModuleManaged,
13674                    FrameSink::new(tx),
13675                )
13676                .unwrap();
13677            self.registry
13678                .register_with_control_ops(
13679                    manifest,
13680                    subc_protocol::PROTOCOL_VERSION,
13681                    connection,
13682                    if health {
13683                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13684                    } else {
13685                        vec![]
13686                    },
13687                )
13688                .unwrap();
13689        }
13690
13691        async fn probe(&mut self) -> crate::router::OutboundFrame {
13692            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13693                Ok(frame) => Some(frame),
13694                Err(mpsc::error::TryRecvError::Empty) => None,
13695                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13696            })
13697            .await;
13698            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13699            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13700            frame
13701        }
13702
13703        fn no_probe(&mut self) {
13704            assert!(matches!(
13705                self.rx.try_recv(),
13706                Err(mpsc::error::TryRecvError::Empty)
13707            ));
13708        }
13709
13710        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13711            self.forwarding
13712                .complete_module_control_rpc(
13713                    ConnectionId::new(connection),
13714                    frame.header.corr,
13715                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13716                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13717                        status: HealthStatus::Ok,
13718                        detail: None,
13719                        metrics: None,
13720                    }),
13721                )
13722                .unwrap();
13723        }
13724    }
13725
13726    #[tokio::test(start_paused = true)]
13727    async fn no_health_children_park_for_sixty_seconds() {
13728        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13729        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13730        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13731        unadvertised.hello(1, false);
13732        // Include any one-off registration wake in the budget; receiving a
13733        // no-health HELLO must not turn a parked child into a polling child.
13734        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13735        // a single 60 s jump would only observe one expired timer.
13736        for _ in 0..6000 {
13737            tokio::time::advance(Duration::from_millis(10)).await;
13738            tokio::task::yield_now().await;
13739        }
13740        for actor in [&none, &unregistered, &unadvertised] {
13741            assert!(
13742                actor.turns() <= 5,
13743                "no-health actor woke {} times",
13744                actor.turns()
13745            );
13746            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13747        }
13748    }
13749
13750    #[tokio::test(start_paused = true)]
13751    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13752        // With a zero cadence the first probe is due at once, so this measures
13753        // only how fast the registration notification starts health probing,
13754        // not the normal 30-33 s wait before a first probe (checked below).
13755        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13756        let tick = Instant::now();
13757        actor.hello(1, true);
13758        actor.probe().await;
13759        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13760    }
13761
13762    #[tokio::test(start_paused = true)]
13763    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13764        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13765        let hello_tick = Instant::now();
13766        actor.hello(1, true);
13767        let armed = actor
13768            .wait_for_select(|checkpoint| {
13769                checkpoint.registered_connection == Some(ConnectionId::new(1))
13770                    && checkpoint.next_probe_at.is_some()
13771            })
13772            .await;
13773        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13774        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13775        let first_deadline = hello_tick + delay;
13776        assert_eq!(armed.next_probe_at, Some(first_deadline));
13777        tokio::time::advance(Duration::from_secs(29)).await;
13778        let turns = actor.turns();
13779        actor
13780            .registry
13781            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13782            .unwrap();
13783        let updated = actor
13784            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13785            .await;
13786        assert_eq!(
13787            updated.next_probe_at,
13788            Some(first_deadline),
13789            "catalog update must not reset cadence"
13790        );
13791        actor.no_probe();
13792        tokio::time::advance(first_deadline - Instant::now()).await;
13793        let frame = actor.probe().await;
13794        actor.answer(1, frame);
13795        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13796        let next_deadline = Instant::now() + next;
13797        let rearmed = actor
13798            .wait_for_select(|checkpoint| {
13799                checkpoint
13800                    .next_probe_at
13801                    .is_some_and(|deadline| deadline > first_deadline)
13802            })
13803            .await;
13804        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13805        tokio::time::advance(next - Duration::from_nanos(1)).await;
13806        actor.no_probe();
13807        tokio::time::advance(Duration::from_nanos(1)).await;
13808        actor.probe().await;
13809    }
13810
13811    #[tokio::test(start_paused = true)]
13812    async fn disconnect_disarms_and_reconnect_rearms_health() {
13813        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13814        actor.hello(1, true);
13815        actor
13816            .wait_for_select(|checkpoint| {
13817                checkpoint.registered_connection == Some(ConnectionId::new(1))
13818                    && checkpoint.next_probe_at.is_some()
13819            })
13820            .await;
13821        let turns = actor.turns();
13822        let tick = Instant::now();
13823        actor
13824            .registry
13825            .deregister_connection(ConnectionId::new(1))
13826            .unwrap();
13827        let disarmed = actor
13828            .wait_for_select(|checkpoint| {
13829                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13830            })
13831            .await;
13832        assert_eq!(disarmed.next_probe_at, None);
13833        assert_eq!(disarmed.wake_after, None);
13834        assert_eq!(
13835            Instant::now(),
13836            tick,
13837            "disconnect must disarm in the notification tick"
13838        );
13839        tokio::time::advance(Duration::from_secs(60)).await;
13840        actor.no_probe();
13841        let reconnect_tick = Instant::now();
13842        actor.hello(2, true);
13843        let rearmed = actor
13844            .wait_for_select(|checkpoint| {
13845                checkpoint.registered_connection == Some(ConnectionId::new(2))
13846                    && checkpoint.next_probe_at.is_some()
13847            })
13848            .await;
13849        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13850        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13851        tokio::time::advance(delay).await;
13852        actor.probe().await;
13853    }
13854
13855    #[tokio::test(start_paused = true)]
13856    async fn reregistration_without_health_disarms_the_old_schedule() {
13857        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13858        actor.hello(1, true);
13859        actor
13860            .wait_for_select(|checkpoint| {
13861                checkpoint.registered_connection == Some(ConnectionId::new(1))
13862                    && checkpoint.next_probe_at.is_some()
13863            })
13864            .await;
13865        actor
13866            .registry
13867            .deregister_connection(ConnectionId::new(1))
13868            .unwrap();
13869        actor.hello(2, false);
13870        let disarmed = actor
13871            .wait_for_select(|checkpoint| {
13872                checkpoint.registered_connection == Some(ConnectionId::new(2))
13873            })
13874            .await;
13875        assert_eq!(disarmed.next_probe_at, None);
13876        assert_eq!(disarmed.wake_after, None);
13877        tokio::time::advance(Duration::from_secs(60)).await;
13878        actor.no_probe();
13879    }
13880
13881    #[tokio::test(start_paused = true)]
13882    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13883        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13884        actor.hello(1, true);
13885        let armed = actor
13886            .wait_for_select(|checkpoint| {
13887                checkpoint.registered_connection == Some(ConnectionId::new(1))
13888                    && checkpoint.next_probe_at.is_some()
13889            })
13890            .await;
13891        // Simulate the old process's connection closing before the restart
13892        // tears the process down. The replacement process has started but has
13893        // not sent its HELLO, so it is not registered.
13894        actor
13895            .registry
13896            .deregister_connection(ConnectionId::new(1))
13897            .unwrap();
13898        actor.module.restart(Some(0)).await.unwrap();
13899        let parked = actor
13900            .wait_for_select(|checkpoint| {
13901                checkpoint.generation > armed.generation
13902                    && checkpoint.registered_connection.is_none()
13903                    && checkpoint.next_probe_at.is_none()
13904            })
13905            .await;
13906        assert_eq!(parked.wake_after, None);
13907        actor.rx.close();
13908        // Drain the stop notice the restart queued for the old process's connection,
13909        // so it isn't mistaken for traffic to the replacement.
13910        while actor.rx.try_recv().is_ok() {}
13911        let turns = actor.turns();
13912        tokio::time::advance(Duration::from_secs(60)).await;
13913        let after = actor.turns();
13914        eprintln!(
13915            "restart park turns: before={turns} after={after} delta={}",
13916            after - turns
13917        );
13918        assert_eq!(after, turns, "replacement must park before HELLO");
13919        let hello_tick = Instant::now();
13920        actor.hello(2, true);
13921        let rearmed = actor
13922            .wait_for_select(|checkpoint| {
13923                checkpoint.registered_connection == Some(ConnectionId::new(2))
13924                    && checkpoint.next_probe_at.is_some()
13925            })
13926            .await;
13927        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13928        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13929        tokio::time::advance(delay).await;
13930        actor.probe().await;
13931    }
13932
13933    #[tokio::test(start_paused = true)]
13934    async fn rescan_adds_and_removes_http_health_without_polling() {
13935        use tokio::{
13936            io::{AsyncReadExt, AsyncWriteExt},
13937            net::TcpListener,
13938        };
13939        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13940        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13941        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13942        // One millisecond separates successive probes without a polling delay.
13943        actor.health.cadence = Duration::from_millis(1);
13944        let (started_tx, mut started_rx) = mpsc::channel(8);
13945        let peer = tokio::spawn(async move {
13946            loop {
13947                let (mut socket, _) = listener.accept().await.unwrap();
13948                let mut request = [0; 1024];
13949                assert!(socket.read(&mut request).await.unwrap() > 0);
13950                started_tx.send(()).await.unwrap();
13951                socket
13952                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13953                    .await
13954                    .unwrap();
13955            }
13956        });
13957        let tick = Instant::now();
13958        let turns = actor.turns();
13959        actor
13960            .module
13961            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13962            .await
13963            .unwrap();
13964        let armed = actor
13965            .wait_for_select(|checkpoint| {
13966                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13967            })
13968            .await;
13969        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13970        assert_eq!(Instant::now(), tick);
13971        tokio::time::advance(Duration::from_millis(1)).await;
13972        let mut started = false;
13973        wait_for("successful HTTP probe", || {
13974            started |= started_rx.try_recv().is_ok();
13975            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13976                .then_some(())
13977        })
13978        .await;
13979        assert!(started, "added HTTP check must start at its first deadline");
13980        assert_eq!(
13981            actor.module.status().unwrap().health.status,
13982            SupervisorHealthStatus::Ok
13983        );
13984        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13985        actor.health.http = None;
13986        let turns = actor.turns();
13987        actor
13988            .module
13989            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13990            .await
13991            .unwrap();
13992        let parked = actor
13993            .wait_for_select(|checkpoint| {
13994                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13995            })
13996            .await;
13997        assert_eq!(parked.wake_after, None);
13998        let turns = actor.turns();
13999        tokio::time::advance(Duration::from_secs(60)).await;
14000        assert_eq!(
14001            actor.turns(),
14002            turns,
14003            "removed HTTP check must park the actor"
14004        );
14005        assert!(
14006            started_rx.try_recv().is_err(),
14007            "removed HTTP check must stay disarmed"
14008        );
14009        assert_eq!(
14010            actor.module.status().unwrap().health,
14011            ModuleHealthStatus::default()
14012        );
14013        peer.abort();
14014    }
14015}
14016
14017#[cfg(test)]
14018mod drain_busy_reprobe_tests {
14019    //! The drain's busy-gauge wait, driven against an in-process module
14020    //! connection whose health answers the test scripts. Time is paused, so
14021    //! every elapsed duration below is exact virtual time.
14022    use super::*;
14023    use std::collections::BTreeMap;
14024    use std::sync::Mutex as StdMutex;
14025    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14026
14027    const MODULE_ID: &str = "drain-reprobe-module";
14028    const CONNECTION: u64 = 7;
14029    const GAUGE: &str = "runs_in_flight";
14030    /// The production default for both the health cadence and the drain
14031    /// ceiling, which is the pairing that used to cost a full ceiling.
14032    const CADENCE: Duration = Duration::from_secs(30);
14033    const CEILING: Duration = Duration::from_secs(30);
14034
14035    /// How the scripted module answers each `health.check`.
14036    #[derive(Clone, Copy)]
14037    enum Answer {
14038        /// Report the gauge busy until this instant, quiet from then on, after
14039        /// taking `latency` to answer.
14040        BusyUntil { until: Instant, latency: Duration },
14041        /// Answer at once, with metrics that leave the declared gauge out.
14042        OmitsGauge,
14043        /// Never answer.
14044        Silent,
14045    }
14046
14047    struct Drain {
14048        forwarding: Arc<ForwardingTable>,
14049        runtime: SupervisorRuntimeConfig,
14050        endpoint: crate::ModuleEndpointId,
14051        /// When each probe reached the module.
14052        probes: Arc<StdMutex<Vec<Instant>>>,
14053        responder: tokio::task::JoinHandle<()>,
14054    }
14055
14056    impl Drain {
14057        fn start(answer: Answer) -> Self {
14058            let registry = Arc::new(Registry::default());
14059            let forwarding = Arc::new(ForwardingTable::default());
14060            let health = HealthConfig {
14061                cadence: CADENCE,
14062                ..HealthConfig::default()
14063            };
14064            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14065                .with_forwarding(forwarding.clone())
14066                .with_health_config(health)
14067                .runtime_config();
14068            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14069            let endpoint = forwarding
14070                .register_module_connection(
14071                    ConnectionId::new(CONNECTION),
14072                    MODULE_ID.to_string(),
14073                    subc_protocol::PROTOCOL_VERSION,
14074                    Concurrency::ModuleManaged,
14075                    FrameSink::new(tx),
14076                )
14077                .unwrap();
14078            let probes = Arc::new(StdMutex::new(Vec::new()));
14079            let responder = tokio::spawn({
14080                let forwarding = forwarding.clone();
14081                let probes = probes.clone();
14082                async move {
14083                    while let Some(frame) = rx.recv().await {
14084                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14085                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14086                        probes.lock().unwrap().push(Instant::now());
14087                        let metrics = match answer {
14088                            Answer::BusyUntil { until, latency } => {
14089                                sleep(latency).await;
14090                                let busy = u64::from(Instant::now() < until);
14091                                serde_json::json!({ GAUGE: busy })
14092                            }
14093                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14094                            Answer::Silent => continue,
14095                        };
14096                        // A probe abandoned at its deadline has no RPC left to
14097                        // complete, so a late answer may be refused.
14098                        let _ = forwarding.complete_module_control_rpc(
14099                            ConnectionId::new(CONNECTION),
14100                            frame.header.corr,
14101                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14102                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14103                                status: HealthStatus::Ok,
14104                                detail: None,
14105                                metrics: Some(metrics),
14106                            }),
14107                        );
14108                    }
14109                }
14110            });
14111            Self {
14112                forwarding,
14113                runtime,
14114                endpoint,
14115                probes,
14116                responder,
14117            }
14118        }
14119
14120        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14121        /// how long it took.
14122        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14123            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14124            (drained, elapsed)
14125        }
14126
14127        /// [`Self::run`], also returning what the drain's probes read.
14128        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14129            let started = Instant::now();
14130            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14131            let drained = wait_for_forwarding_quiescence(
14132                &self.forwarding,
14133                MODULE_ID,
14134                &self.runtime,
14135                self.endpoint,
14136                started + ceiling,
14137                &[GAUGE.to_string()],
14138                DrainScope::Active,
14139                &mut trace,
14140            )
14141            .await
14142            .unwrap();
14143            (drained, started.elapsed(), trace)
14144        }
14145
14146        fn probes(&self) -> Vec<Instant> {
14147            self.probes.lock().unwrap().clone()
14148        }
14149    }
14150
14151    impl Drop for Drain {
14152        fn drop(&mut self) {
14153            self.responder.abort();
14154        }
14155    }
14156
14157    #[tokio::test(start_paused = true)]
14158    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14159        let drain = Drain::start(Answer::BusyUntil {
14160            until: Instant::now() + Duration::from_secs(1),
14161            latency: Duration::ZERO,
14162        });
14163        let (drained, elapsed) = drain.run(CEILING).await;
14164        assert!(
14165            drained,
14166            "gauges that read zero during the drain must let it finish drained, \
14167             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14168        );
14169        assert!(
14170            elapsed < Duration::from_secs(3),
14171            "drain took {elapsed:?} after the gauges went quiet at 1s"
14172        );
14173        let probes = drain.probes();
14174        assert!(
14175            probes.len() >= 2,
14176            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14177        );
14178    }
14179
14180    /// The same shape in real time on a multi-threaded runtime: the first
14181    /// probe is answered busy at once, within a millisecond of the drain
14182    /// starting, and later probes read idle. The drain must keep probing
14183    /// after that first instant answer and end soon after the gauge falls.
14184    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14185    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14186        let drain = Drain::start(Answer::BusyUntil {
14187            until: Instant::now() + Duration::from_millis(600),
14188            latency: Duration::ZERO,
14189        });
14190        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14191        assert!(drained, "the gauge fell at 600ms: {trace}");
14192        assert!(
14193            elapsed < Duration::from_secs(3),
14194            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14195        );
14196        let probes = drain.probes();
14197        assert!(
14198            probes.len() >= 2 && trace.probes as usize == probes.len(),
14199            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14200        );
14201    }
14202
14203    #[tokio::test(start_paused = true)]
14204    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14205        let started = Instant::now();
14206        let drain = Drain::start(Answer::BusyUntil {
14207            until: started + Duration::from_secs(3600),
14208            latency: Duration::ZERO,
14209        });
14210        let (drained, elapsed) = drain.run(CEILING).await;
14211        assert!(!drained, "a module busy throughout must not drain");
14212        assert!(
14213            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14214            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14215        );
14216        // Re-probing is paced by the interval, not a busy loop: at most one
14217        // probe per interval across the ceiling, plus the first at time zero.
14218        let probes = drain.probes();
14219        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14220        assert!(
14221            probes.len() as u128 <= max_probes,
14222            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14223            probes.len()
14224        );
14225        for pair in probes.windows(2) {
14226            assert!(
14227                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14228                "consecutive probes {:?} apart",
14229                pair[1] - pair[0]
14230            );
14231        }
14232    }
14233
14234    #[tokio::test(start_paused = true)]
14235    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14236        // Each answer takes longer than the re-probe interval; the next probe
14237        // must wait for the previous answer and then the interval.
14238        let latency = Duration::from_secs(1);
14239        let drain = Drain::start(Answer::BusyUntil {
14240            until: Instant::now() + Duration::from_secs(3600),
14241            latency,
14242        });
14243        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14244        assert!(!drained);
14245        let probes = drain.probes();
14246        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14247        for pair in probes.windows(2) {
14248            assert!(
14249                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14250                "a probe started {:?} after the previous one, before it could finish",
14251                pair[1] - pair[0]
14252            );
14253        }
14254    }
14255
14256    #[tokio::test(start_paused = true)]
14257    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14258        let drain = Drain::start(Answer::Silent);
14259        let ceiling = Duration::from_secs(2);
14260        let (drained, elapsed) = drain.run(ceiling).await;
14261        assert!(!drained, "an unanswered probe counts as busy");
14262        assert!(
14263            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14264            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14265        );
14266        assert_eq!(
14267            drain.probes().len(),
14268            1,
14269            "the unanswered probe holds the only slot until the ceiling"
14270        );
14271    }
14272
14273    /// Every event logged while it is installed, as `message` plus fields.
14274    #[derive(Clone, Default)]
14275    struct LogCapture {
14276        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14277    }
14278
14279    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14280        fn on_event(
14281            &self,
14282            event: &tracing::Event<'_>,
14283            _context: tracing_subscriber::layer::Context<'_, S>,
14284        ) {
14285            struct Fields(BTreeMap<String, String>);
14286            impl tracing::field::Visit for Fields {
14287                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14288                    self.0
14289                        .insert(field.name().to_string(), format!("{value:?}"));
14290                }
14291            }
14292            let mut fields = Fields(BTreeMap::new());
14293            event.record(&mut fields);
14294            self.events.lock().unwrap().push(fields.0);
14295        }
14296    }
14297
14298    impl LogCapture {
14299        fn install(&self) -> tracing::subscriber::DefaultGuard {
14300            use tracing_subscriber::layer::SubscriberExt;
14301            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14302        }
14303
14304        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14305            self.events
14306                .lock()
14307                .unwrap()
14308                .iter()
14309                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14310                .cloned()
14311                .collect()
14312        }
14313    }
14314
14315    /// A module that declares a busy gauge but leaves it out of its health
14316    /// answers holds a drain to its ceiling. The timeout line must say so by
14317    /// name, and the drain must warn about the missing gauge once, not on
14318    /// every probe.
14319    #[tokio::test(start_paused = true)]
14320    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14321        let capture = LogCapture::default();
14322        let _guard = capture.install();
14323        let drain = Drain::start(Answer::OmitsGauge);
14324        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14325        assert!(!drained, "a gauge never reported cannot read quiet");
14326        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14327
14328        let missing = capture.with_message("lacks a declared busy gauge");
14329        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14330        assert_eq!(
14331            missing[0].get("missing_gauges").map(String::as_str),
14332            Some(GAUGE)
14333        );
14334
14335        warn_drain_timed_out(
14336            MODULE_ID,
14337            Duration::from_secs(2),
14338            RouteCloseReason::Restart,
14339            &Default::default(),
14340            &trace,
14341        );
14342        let timeouts = capture.with_message("route drain timed out");
14343        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14344        let gauges = &timeouts[0]["busy_gauges"];
14345        assert!(
14346            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14347            "the timeout line must name the gauge and why it read busy: {gauges}"
14348        );
14349        assert!(
14350            gauges.ends_with("; no value ever read"),
14351            "no value was ever read for the gauge: {gauges}"
14352        );
14353        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14354    }
14355
14356    /// The trace names a busy value and when it was read. For a probe that
14357    /// never got an answer, it names the probe's error kind and says that no
14358    /// value was ever read.
14359    #[tokio::test(start_paused = true)]
14360    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14361        let busy = Drain::start(Answer::BusyUntil {
14362            until: Instant::now() + Duration::from_secs(3600),
14363            latency: Duration::ZERO,
14364        });
14365        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14366        let rendered = trace.to_string();
14367        assert!(
14368            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14369                && rendered.contains("; last value non_zero(1) @"),
14370            "{rendered}"
14371        );
14372
14373        let silent = Drain::start(Answer::Silent);
14374        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14375        // The probe is capped at the 1 s ceiling, and its reading is stamped
14376        // when it gave up.
14377        assert_eq!(
14378            trace.to_string(),
14379            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14380        );
14381        assert_eq!(trace.probes, 1);
14382    }
14383
14384    /// A gauge value that is not a count reads busy and is shown as sent.
14385    #[test]
14386    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14387        let gauges = [GAUGE.to_string()];
14388        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14389        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14390        assert_eq!(
14391            busy_gauge_observation(&verdicts),
14392            BusyGaugeObservation::Busy
14393        );
14394        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14395        assert_eq!(
14396            busy_gauge_observation(&zero),
14397            BusyGaugeObservation::Quiescent
14398        );
14399        let none = gauge_verdicts(None, &gauges);
14400        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14401    }
14402}
14403
14404#[cfg(test)]
14405mod health_evidence_tests {
14406    use super::{HealthProbeError, HealthProbeEvidence};
14407    use std::collections::HashSet;
14408
14409    /// The evidential asymmetry, asserted rather than described.
14410    ///
14411    /// Exactly ONE observation is proof a module cannot serve, and the one that
14412    /// fires under CPU starvation is not it. Before the split, all fifteen
14413    /// construction sites collapsed into a single String, so a timeout carried the
14414    /// same weight as a dead lane -- which is how a healthy module was restarted
14415    /// three times in one day.
14416    #[test]
14417    fn only_a_dead_lane_is_proof_of_death() {
14418        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14419        // Three non-proof classes, each for a different reason: silence is
14420        // consistent with health, a bad answer proves the module ALIVE, and a
14421        // daemon-side fault never reached the module at all.
14422        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14423        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14424        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14425    }
14426
14427    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14428    ///
14429    /// A shared label renders two different observations identically in the line an
14430    /// operator reads after an unexplained restart -- the exact confusion this
14431    /// change removes.
14432    #[test]
14433    fn every_evidence_class_has_a_distinct_label() {
14434        let labels = [
14435            HealthProbeError::lane_dead("").label(),
14436            HealthProbeError::no_answer("").label(),
14437            HealthProbeError::bad_answer("").label(),
14438            HealthProbeError::misconfigured("").label(),
14439        ];
14440        let unique: HashSet<_> = labels.iter().collect();
14441        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14442    }
14443
14444    /// The class is additional information, not a replacement.
14445    ///
14446    /// An operator needs both "this was silence" and the specific text saying how
14447    /// long we waited; a classification that swallowed the message would trade one
14448    /// missing distinction for another.
14449    #[test]
14450    fn classification_preserves_the_original_message() {
14451        let err = HealthProbeError::no_answer("module did not answer within 5s");
14452        assert_eq!(err.to_string(), "module did not answer within 5s");
14453        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14454    }
14455}
14456
14457#[cfg(test)]
14458mod health_tombstone_tests {
14459    use std::{path::PathBuf, sync::Arc, time::Duration};
14460
14461    use subc_protocol::{
14462        manifest::Concurrency,
14463        session::{HealthStatus, ModuleControlResponse},
14464    };
14465    use tokio::sync::mpsc;
14466
14467    use super::{
14468        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14469        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14470    };
14471    use crate::{
14472        control::ControlHandler,
14473        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14474        registry::{ConnectionId, Registry},
14475        router::FrameSink,
14476    };
14477
14478    struct ProbeHarness {
14479        spec: ModuleSpec,
14480        runtime: SupervisorRuntimeConfig,
14481        forwarding: Arc<ForwardingTable>,
14482        module_connection: ConnectionId,
14483        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14484        handler: ControlHandler,
14485        module: super::SupervisedModule,
14486    }
14487
14488    fn probe_harness() -> ProbeHarness {
14489        let registry = Arc::new(Registry::default());
14490        let forwarding = Arc::new(ForwardingTable::default());
14491        let supervisor_handle = super::SupervisorHandle::new();
14492        let health = HealthConfig {
14493            http: None,
14494            cadence: Duration::from_secs(30),
14495            deadline: Duration::from_secs(5),
14496            failure_threshold: 3,
14497            on_degraded: HealthAction::Report,
14498            on_failing: HealthAction::Report,
14499            critical: false,
14500        };
14501        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14502            .with_forwarding(Arc::clone(&forwarding))
14503            .with_handle(supervisor_handle.clone())
14504            .with_health_config(health);
14505        let spec = ModuleSpec {
14506            module_id: "late-health-module".to_string(),
14507            program: PathBuf::from("disabled-module"),
14508            args: Vec::new(),
14509            env: Vec::new(),
14510            reserved: false,
14511            reserved_prefixes: Vec::new(),
14512            protocol: ModuleProtocol::Subc,
14513            overlap: Default::default(),
14514        };
14515        let module = supervisor
14516            .supervise_configured(spec.clone(), false)
14517            .unwrap();
14518        let runtime = supervisor.runtime_config();
14519        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14520            .with_supervisor(supervisor_handle);
14521        let module_connection = ConnectionId::new(700);
14522        let (module_tx, module_rx) = mpsc::channel(8);
14523        forwarding
14524            .register_module_connection(
14525                module_connection,
14526                spec.module_id.clone(),
14527                subc_protocol::PROTOCOL_VERSION,
14528                Concurrency::ModuleManaged,
14529                FrameSink::new(module_tx),
14530            )
14531            .unwrap();
14532
14533        ProbeHarness {
14534            spec,
14535            runtime,
14536            forwarding,
14537            module_connection,
14538            module_rx,
14539            handler,
14540            module,
14541        }
14542    }
14543
14544    async fn finish_after(
14545        harness: &mut ProbeHarness,
14546        stall: Duration,
14547    ) -> ModuleControlRpcCompletion {
14548        assert!(stall > harness.runtime.health.deadline);
14549        let deadline = harness.runtime.health.deadline;
14550        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14551        let answer = async {
14552            let frame = harness.module_rx.recv().await.expect("health.check frame");
14553            tokio::time::advance(deadline).await;
14554            tokio::task::yield_now().await;
14555            tokio::time::advance(stall - deadline).await;
14556            harness
14557                .forwarding
14558                .complete_module_control_rpc(
14559                    harness.module_connection,
14560                    frame.header.corr,
14561                    Some("health.check"),
14562                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14563                        status: HealthStatus::Ok,
14564                        detail: None,
14565                        metrics: None,
14566                    }),
14567                )
14568                .unwrap()
14569        };
14570        let (probe_result, completion) = tokio::join!(probe, answer);
14571        let err = probe_result.expect_err("probe must miss its deadline");
14572        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14573        completion
14574    }
14575
14576    async fn time_out_without_answer(harness: &mut ProbeHarness) {
14577        let deadline = harness.runtime.health.deadline;
14578        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14579        let exhaust_deadline = async {
14580            let _frame = harness.module_rx.recv().await.expect("health.check frame");
14581            tokio::time::advance(deadline).await;
14582            tokio::task::yield_now().await;
14583        };
14584        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14585        let err = probe_result.expect_err("probe must miss its deadline");
14586        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14587    }
14588
14589    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14590        let registry = Arc::clone(&harness.module.inner.registry);
14591        let snapshot = Arc::clone(&harness.module.inner.snapshot);
14592        let process_liveness = super::SupervisorProcessLiveness::default();
14593        let mut child = None;
14594        let cycle = super::run_health_probe_cycle(
14595            &harness.spec,
14596            &harness.runtime,
14597            &registry,
14598            &process_liveness,
14599            &snapshot,
14600            &mut child,
14601        );
14602        let peer = async {
14603            let frame = harness.module_rx.recv().await.expect("health.check frame");
14604            if answer {
14605                harness
14606                    .forwarding
14607                    .complete_module_control_rpc(
14608                        harness.module_connection,
14609                        frame.header.corr,
14610                        Some("health.check"),
14611                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14612                            status: HealthStatus::Ok,
14613                            detail: None,
14614                            metrics: Some(serde_json::json!({"ready": true})),
14615                        }),
14616                    )
14617                    .unwrap();
14618            } else {
14619                tokio::time::advance(harness.runtime.health.deadline).await;
14620                tokio::task::yield_now().await;
14621            }
14622        };
14623        tokio::join!(cycle, peer);
14624    }
14625
14626    #[tokio::test(start_paused = true)]
14627    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14628        let mut harness = probe_harness();
14629        // Drive the probe cycle directly with an in-memory wire peer. Stop the
14630        // disabled module's monitor so only this test owns lifecycle transitions;
14631        // no OS process is launched, and a restart is observed at scheduling.
14632        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14633        monitor.abort();
14634        let _ = monitor.await;
14635        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14636            state.enabled = true;
14637            state.state = super::ModuleState::Running;
14638            state.process_alive = true;
14639        })
14640        .unwrap();
14641
14642        run_probe_cycle(&mut harness, true).await;
14643        assert_eq!(
14644            harness.module.status().unwrap().health.status,
14645            super::SupervisorHealthStatus::Ok
14646        );
14647
14648        for failures in 1..harness.runtime.health.failure_threshold {
14649            run_probe_cycle(&mut harness, false).await;
14650            let status = harness.module.status().unwrap();
14651            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14652            assert_eq!(status.health.consecutive_failures, failures);
14653            assert!(status.health.last_probe_ms.is_some());
14654            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14655            assert!(status.health.metrics.is_none());
14656            assert_eq!(status.state, super::ModuleState::Running);
14657            assert!(status.process_alive);
14658            assert_eq!(status.restart_count, 0);
14659            assert_eq!(status.lifetime_restarts, 0);
14660            assert!(status.health.last_action.is_none());
14661            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14662        }
14663
14664        run_probe_cycle(&mut harness, true).await;
14665        let recovered = harness.module.status().unwrap();
14666        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14667        assert_eq!(recovered.health.consecutive_failures, 0);
14668        assert!(recovered.health.detail.is_none());
14669        assert_eq!(
14670            recovered.health.metrics,
14671            Some(serde_json::json!({"ready": true}))
14672        );
14673        assert_eq!(recovered.lifetime_restarts, 0);
14674
14675        for failures in 1..=harness.runtime.health.failure_threshold {
14676            run_probe_cycle(&mut harness, false).await;
14677            let status = harness.module.status().unwrap();
14678            assert_eq!(status.health.consecutive_failures, failures);
14679            if failures < harness.runtime.health.failure_threshold {
14680                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14681                assert_eq!(status.state, super::ModuleState::Running);
14682                assert_eq!(status.lifetime_restarts, 0);
14683                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14684            } else {
14685                assert_eq!(
14686                    status.health.status,
14687                    super::SupervisorHealthStatus::Unresponsive
14688                );
14689                assert_eq!(status.state, super::ModuleState::Restarting);
14690                assert_eq!(status.restart_count, 1);
14691                assert_eq!(status.lifetime_restarts, 1);
14692                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14693                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14694            }
14695        }
14696    }
14697
14698    #[tokio::test(start_paused = true)]
14699    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14700        let mut harness = probe_harness();
14701
14702        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14703        let first_latency = match &first {
14704            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14705            other => panic!("late answer was not retained: {other:?}"),
14706        };
14707        assert!(harness.handler.observe_module_control_completion(first));
14708
14709        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14710        let second_latency = match &second {
14711            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14712            other => panic!("late answer was not retained: {other:?}"),
14713        };
14714        assert!(harness.handler.observe_module_control_completion(second));
14715
14716        assert_eq!(first_latency, Duration::from_secs(8));
14717        assert_eq!(
14718            second_latency - first_latency,
14719            Duration::from_secs(3),
14720            "latency must grow linearly with the additional stall"
14721        );
14722        let health = harness.module.status().unwrap().health;
14723        assert_eq!(health.late_answer_count, 2);
14724        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14725    }
14726
14727    /// A module that answers every probe late must never march to the kill
14728    /// threshold: the late answer proves it is alive, so it must clear the miss
14729    /// streak the timeout recorded. Without the reset, a CPU-starved module
14730    /// that serves every probe seconds past the deadline accumulates
14731    /// `consecutive_failures` to the threshold and is killed — the exact
14732    /// sequence from the 2026-08-14 aft disable, where the daemon logged
14733    /// "proves the module is alive" five times while counting five misses.
14734    #[tokio::test(start_paused = true)]
14735    async fn late_answer_clears_the_consecutive_failure_streak() {
14736        let mut harness = probe_harness();
14737
14738        // Timeout recorded first: the probe path saw no answer in time.
14739        time_out_without_answer(&mut harness).await;
14740        harness
14741            .module
14742            .record_health_probe_failure_for_test("[no-answer] test miss")
14743            .unwrap();
14744        assert_eq!(
14745            harness.module.status().unwrap().health.consecutive_failures,
14746            1,
14747            "precondition: the miss must be on the streak before the late answer"
14748        );
14749
14750        // The stalled reply then lands: proof of life.
14751        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14752        assert!(matches!(
14753            late,
14754            ModuleControlRpcCompletion::LateHealthAnswer { .. }
14755        ));
14756        assert!(harness.handler.observe_module_control_completion(late));
14757
14758        let health = harness.module.status().unwrap().health;
14759        assert_eq!(
14760            health.consecutive_failures, 0,
14761            "a late answer is an answer: the streak must reset"
14762        );
14763        assert_eq!(health.late_answer_count, 1);
14764    }
14765
14766    #[tokio::test(start_paused = true)]
14767    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14768        let mut harness = probe_harness();
14769
14770        for _ in 0..20 {
14771            time_out_without_answer(&mut harness).await;
14772            assert_eq!(
14773                harness.forwarding.health_probe_tombstone_count().unwrap(),
14774                1
14775            );
14776        }
14777    }
14778}
14779
14780#[cfg(test)]
14781mod child_env_tests {
14782    #[cfg(windows)]
14783    use super::{
14784        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14785    };
14786    #[cfg(windows)]
14787    use std::sync::{Arc, OnceLock};
14788    #[cfg(windows)]
14789    #[test]
14790    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14791        let spec = spec(Vec::new());
14792        let mut command = Command::new("unused");
14793        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14794            .unwrap()
14795            .unwrap();
14796        let name = command
14797            .as_std()
14798            .get_envs()
14799            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14800            .and_then(|(_, value)| value)
14801            .unwrap();
14802        assert_eq!(name, OsStr::new(handoff.name()));
14803        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14804        assert!(command.as_std().get_envs().any(|(key, value)| key
14805            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14806            && value == Some(OsStr::new("env"))));
14807        assert!(command
14808            .as_std()
14809            .get_envs()
14810            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14811    }
14812
14813    #[cfg(windows)]
14814    #[test]
14815    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14816        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14817        snapshot.process_alive = true;
14818        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14819        assert_eq!(
14820            observed_launch_nonce_source(&snapshot).as_deref(),
14821            Some("env")
14822        );
14823        let consumed = Arc::new(OnceLock::new());
14824        snapshot.pipe_consumed = Some(consumed.clone());
14825        assert_eq!(
14826            observed_launch_nonce_source(&snapshot).as_deref(),
14827            Some("env")
14828        );
14829        consumed.set(()).unwrap();
14830        assert_eq!(
14831            observed_launch_nonce_source(&snapshot).as_deref(),
14832            Some("pipe")
14833        );
14834        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14835        assert_eq!(
14836            observed_launch_nonce_source(&snapshot).as_deref(),
14837            Some("env")
14838        );
14839        clear_current_process_facts(&mut snapshot);
14840        assert_eq!(observed_launch_nonce_source(&snapshot), None);
14841    }
14842    use super::{
14843        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14844        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14845        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14846    };
14847    use std::{ffi::OsStr, path::PathBuf};
14848    use tokio::process::Command;
14849
14850    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14851        ModuleSpec {
14852            module_id: "env-plan".to_string(),
14853            program: PathBuf::from("/nonexistent"),
14854            args: Vec::new(),
14855            env,
14856            reserved: false,
14857            reserved_prefixes: Vec::new(),
14858            protocol: ModuleProtocol::Subc,
14859            overlap: Default::default(),
14860        }
14861    }
14862
14863    #[test]
14864    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14865        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14866        let digest = LaunchSpecDigest::from_spec(&base);
14867        let mut reordered = base.clone();
14868        reordered.env.reverse();
14869        reordered.module_id = "different-id".into();
14870        reordered.reserved = true;
14871        reordered.overlap = Default::default();
14872        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14873        let mut delimiter_collision = base.clone();
14874        delimiter_collision.env[0] = ("A=B".into(), "C".into());
14875        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14876        let mut changed = base.clone();
14877        changed.protocol = ModuleProtocol::None;
14878        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14879        changed = base.clone();
14880        changed.args.push("--flag".into());
14881        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14882        changed = base.clone();
14883        changed.program = PathBuf::from("/other");
14884        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14885        changed = base.clone();
14886        changed.env[1].1 = "other".into();
14887        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14888        assert!(!format!("{digest:?}").contains("B=C"));
14889        assert!(!format!("{digest:?}").contains("[u8"));
14890    }
14891
14892    #[cfg(unix)]
14893    #[test]
14894    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14895        use std::os::unix::ffi::OsStringExt;
14896        let mut left = spec(Vec::new());
14897        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14898        let mut right = left.clone();
14899        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14900        assert_eq!(
14901            left.program.to_string_lossy(),
14902            right.program.to_string_lossy()
14903        );
14904        assert_ne!(
14905            LaunchSpecDigest::from_spec(&left),
14906            LaunchSpecDigest::from_spec(&right)
14907        );
14908    }
14909
14910    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
14911    /// one still gets its own.
14912    ///
14913    /// This is the narrow goal `env_clear()` was reached for, and the reason the
14914    /// fix is `env_remove` rather than deleting the line: an operator's ambient
14915    /// filter silently becoming an unconfigured module's log level is a real
14916    /// defect, just a much smaller one than clearing the environment.
14917    ///
14918    /// Asserted on the command plan rather than a spawned child because proving
14919    /// the ABSENCE of an inherited variable needs the parent's environment
14920    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
14921    /// removal as `(key, None)`, which is exactly the distinction wanted: not
14922    /// "absent because nobody set it" but "explicitly unset for the child".
14923    #[test]
14924    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14925        let mut command = Command::new("/nonexistent");
14926        apply_child_env(&mut command, &spec(Vec::new()));
14927        let removed = command
14928            .as_std()
14929            .get_envs()
14930            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14931        assert!(
14932            removed,
14933            "ambient CK_LOG must be explicitly removed for an unconfigured module"
14934        );
14935
14936        let mut configured = Command::new("/nonexistent");
14937        apply_child_env(
14938            &mut configured,
14939            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14940        );
14941        let effective = configured
14942            .as_std()
14943            .get_envs()
14944            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14945            .last()
14946            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14947        assert_eq!(
14948            effective,
14949            Some(Some("debug".to_string())),
14950            "a module's configured CK_LOG must survive the ambient removal"
14951        );
14952    }
14953
14954    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
14955    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
14956    /// the same reason as the CK_LOG test above.
14957    ///
14958    /// The argument is the load-bearing half: a stock binary exits on an
14959    /// unknown flag before it listens, so with `--subc` appended the mode
14960    /// could not supervise the one process it exists for. Found by the first
14961    /// conformance run (nats-server: `flag provided but not defined: -subc`).
14962    #[test]
14963    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14964        let connection_file = std::path::Path::new("/run/subc-connection.json");
14965        let handle = SupervisorHandle::new();
14966
14967        let mut none_spec = spec(Vec::new());
14968        none_spec.protocol = ModuleProtocol::None;
14969        let mut none = Command::new("/nonexistent");
14970        let none_handoff =
14971            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14972                .expect("protocol-none spawn args apply");
14973        assert!(
14974            none_handoff.is_none(),
14975            "protocol:none spawn must not receive a nonce descriptor"
14976        );
14977        #[cfg(windows)]
14978        assert!(!none.as_std().get_envs().any(|(key, value)| key
14979            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14980            && value.is_some()));
14981        assert!(
14982            !none.as_std().get_envs().any(|(key, value)| key
14983                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14984                && value.is_some()),
14985            "protocol:none spawn must not name a nonce descriptor"
14986        );
14987        let none_args: Vec<String> = none
14988            .as_std()
14989            .get_args()
14990            .map(|a| a.to_string_lossy().into_owned())
14991            .collect();
14992        assert!(
14993            !none_args.iter().any(|a| a == SUBC_ARG),
14994            "protocol:none argv must not carry --subc; got {none_args:?}"
14995        );
14996        let none_has_nonce = none
14997            .as_std()
14998            .get_envs()
14999            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15000        assert!(
15001            !none_has_nonce,
15002            "protocol:none spawn must not receive a launch nonce"
15003        );
15004        let none_has_module_id = none
15005            .as_std()
15006            .get_envs()
15007            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15008        assert!(
15009            none_has_module_id,
15010            "SUBC_MODULE_ID is inert and stays on every path"
15011        );
15012        assert!(
15013            handle.spawn_nonce(&none_spec.module_id).is_none(),
15014            "no nonce record for a process that will never present one"
15015        );
15016
15017        // Control: the subc-wire path is unchanged by the branch above.
15018        let wire_spec = spec(Vec::new());
15019        let mut wire = Command::new("/nonexistent");
15020        let wire_handoff =
15021            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15022                .expect("subc-wire spawn args apply");
15023        let wire_fd_env = wire
15024            .as_std()
15025            .get_envs()
15026            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15027            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15028        #[cfg(unix)]
15029        assert_eq!(
15030            wire_fd_env,
15031            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15032            "a subc-wire spawn names the pipe it will receive at descriptor 3"
15033        );
15034        #[cfg(windows)]
15035        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15036        #[cfg(not(any(unix, windows)))]
15037        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15038        let wire_args: Vec<String> = wire
15039            .as_std()
15040            .get_args()
15041            .map(|a| a.to_string_lossy().into_owned())
15042            .collect();
15043        assert_eq!(
15044            wire_args,
15045            vec![
15046                SUBC_ARG.to_string(),
15047                connection_file.to_string_lossy().into_owned()
15048            ],
15049            "a subc-wire spawn still carries --subc <path>"
15050        );
15051        assert_eq!(
15052            wire.as_std()
15053                .get_envs()
15054                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15055            !cfg!(unix),
15056            "only Windows supplies the environment nonce"
15057        );
15058        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15059    }
15060
15061    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
15062    /// spec tries to set it; only a swap candidate carries it.
15063    ///
15064    /// "Set it only on candidates" is not enough, because spawn applies the
15065    /// spec's env verbatim and the daemon's own environment is inherited: either
15066    /// could hand a plain restart the swap role, and a module reading it would
15067    /// warm on its long swap budget while callers wait. Asserted as an explicit
15068    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
15069    /// test above gives.
15070    #[test]
15071    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15072        let role = |command: &Command| {
15073            command
15074                .as_std()
15075                .get_envs()
15076                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15077                .last()
15078                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15079        };
15080        let forged = spec(vec![(
15081            SUBC_SPAWN_ROLE_ENV.to_string(),
15082            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15083        )]);
15084
15085        let mut plain = Command::new("/nonexistent");
15086        apply_child_env(&mut plain, &forged);
15087        apply_spawn_role(&mut plain, SpawnRole::Plain);
15088        assert_eq!(
15089            role(&plain),
15090            Some(None),
15091            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15092        );
15093
15094        let mut candidate = Command::new("/nonexistent");
15095        apply_child_env(&mut candidate, &spec(Vec::new()));
15096        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15097        assert_eq!(
15098            role(&candidate),
15099            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15100        );
15101    }
15102
15103    /// Daemon-private capture retention keys never reach the child.
15104    ///
15105    /// cortexkit-log exposes retention as a Rust struct with no environment
15106    /// names, so these entries are supervisor metadata. Passing them through
15107    /// would invent a public child-process contract by accident.
15108    #[test]
15109    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15110        let mut command = Command::new("/nonexistent");
15111        apply_child_env(
15112            &mut command,
15113            &spec(vec![
15114                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15115                ("KEPT".to_string(), "yes".to_string()),
15116            ]),
15117        );
15118        let keys: Vec<String> = command
15119            .as_std()
15120            .get_envs()
15121            .filter(|(_, value)| value.is_some())
15122            .map(|(key, _)| key.to_string_lossy().into_owned())
15123            .collect();
15124        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15125        assert!(
15126            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15127            "daemon-private capture key leaked to the child: {keys:?}"
15128        );
15129    }
15130}
15131
15132#[cfg(test)]
15133mod jitter_tests {
15134    use super::jittered_health_delay;
15135    use std::{collections::HashSet, time::Duration};
15136
15137    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15138    /// that actually occur rather than invented ones.
15139    ///
15140    /// This is a SAMPLE, not a registry: the property under test is that distinct
15141    /// ids disperse, which holds for any set of distinct strings. Several entries
15142    /// are already historical (modules get renamed), and that costs nothing here --
15143    /// but it means a reader must not mistake this for the live module set, and a
15144    /// rename sweep will match it without there being anything to change.
15145    const FLEET: [&str; 14] = [
15146        "aft",
15147        "alfonso-core",
15148        "magic-context",
15149        "broca",
15150        "thalamus",
15151        "quota",
15152        "engram",
15153        "plexus",
15154        "cerebellum",
15155        "astrocyte",
15156        "synapse",
15157        "subc-mcp",
15158        "cortexkit-credentials",
15159        "subc-federation",
15160    ];
15161
15162    /// Probes must not converge after a fleet-wide restart.
15163    ///
15164    /// This is the property the jitter exists for: every module reconnects at
15165    /// once, and without dispersal all fourteen would then probe on the same
15166    /// tick forever. Nothing failed visibly when this went untested -- a
15167    /// convergent fleet still probes correctly, just in a burst, so the symptom
15168    /// is a periodic load spike that looks like whatever else is running.
15169    #[test]
15170    fn probe_delays_disperse_across_the_fleet() {
15171        let cadence = Duration::from_secs(30);
15172        let delays: HashSet<Duration> = FLEET
15173            .iter()
15174            .map(|id| jittered_health_delay(id, 0, cadence))
15175            .collect();
15176        assert_eq!(
15177            delays.len(),
15178            FLEET.len(),
15179            "every supervised module must land on its own probe offset"
15180        );
15181    }
15182
15183    /// The offset may only ever DELAY a probe, never bring it forward.
15184    ///
15185    /// A delay below the cadence would probe a module more often than
15186    /// configured, which is the opposite of what an operator asked for and
15187    /// would tighten the failure budget without anyone changing it.
15188    #[test]
15189    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15190        let cadence = Duration::from_secs(30);
15191        let span = cadence / 10;
15192        for id in FLEET {
15193            for probe_index in 0..8 {
15194                let delay = jittered_health_delay(id, probe_index, cadence);
15195                assert!(
15196                    delay >= cadence,
15197                    "{id}#{probe_index}: jitter must not shorten the cadence"
15198                );
15199                assert!(
15200                    delay < cadence + span,
15201                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15202                );
15203            }
15204        }
15205    }
15206
15207    /// A module keeps its offset across daemon restarts.
15208    ///
15209    /// The delay is derived rather than randomised precisely so a restart does
15210    /// not re-roll every module into a fresh chance of collision. A random
15211    /// source would satisfy the dispersal test above and quietly lose this.
15212    #[test]
15213    fn a_module_offset_is_stable_across_restarts() {
15214        let cadence = Duration::from_secs(30);
15215        for id in FLEET {
15216            assert_eq!(
15217                jittered_health_delay(id, 0, cadence),
15218                jittered_health_delay(id, 0, cadence),
15219                "{id}: the same module and probe index must produce the same offset"
15220            );
15221        }
15222    }
15223
15224    /// A zero cadence disables probing rather than producing a busy loop.
15225    #[test]
15226    fn zero_cadence_yields_zero_delay() {
15227        assert_eq!(
15228            jittered_health_delay("aft", 0, Duration::ZERO),
15229            Duration::ZERO
15230        );
15231    }
15232}
15233
15234#[cfg(all(test, target_os = "linux"))]
15235mod cgroup_placement_tests {
15236    use super::{
15237        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15238        SuperviseError, SupervisedChild,
15239    };
15240    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15241    use cortexkit_test_support::ScratchDir;
15242    use std::{
15243        fs, io,
15244        path::{Path, PathBuf},
15245        sync::{Arc, Mutex},
15246    };
15247    use tokio::process::Command;
15248
15249    #[tokio::test]
15250    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15251        use super::*;
15252        let dir = ScratchDir::new("unique-spawn-cgroups");
15253        let root = PathBuf::from(format!(
15254            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15255            std::process::id(),
15256            unix_ms_now()
15257        ));
15258        if let Err(error) = fs::create_dir(&root) {
15259            assert!(
15260                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15261                "required cgroup test cannot execute: {error}"
15262            );
15263            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15264            return;
15265        }
15266        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15267        let group_count = || {
15268            fs::read_dir(root.join("subc-modules"))
15269                .unwrap()
15270                .map(|entry| entry.unwrap().file_type().unwrap())
15271                .filter(|kind| kind.is_dir())
15272                .count()
15273        };
15274        let supervisor =
15275            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15276                .with_cgroup_placement(Some(placement.clone()));
15277        let runtime = supervisor.runtime_config();
15278        let mut spec = ModuleSpec {
15279            module_id: "unique-spawn".into(),
15280            program: PathBuf::from("/bin/sleep"),
15281            args: vec!["60".into()],
15282            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15283                .into_iter()
15284                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15285                .collect(),
15286            reserved: false,
15287            reserved_prefixes: vec![],
15288            protocol: ModuleProtocol::None,
15289            overlap: Default::default(),
15290        };
15291        let spawn = |spec: &ModuleSpec| {
15292            spawn_child(
15293                spec,
15294                None,
15295                None,
15296                &runtime.stderr_ring,
15297                None,
15298                &runtime.child_roster,
15299                Some(&placement),
15300            )
15301            .unwrap()
15302        };
15303        let mut live = spawn(&spec);
15304        for _ in 0..3 {
15305            // A new process can enter the old slot while retirement is pending.
15306            let next = spawn(&spec);
15307            assert_ne!(live.module_id, next.module_id);
15308            live.start_kill().unwrap();
15309            live.wait().await.unwrap();
15310            live = next;
15311            assert!(
15312                live.child.try_wait().unwrap().is_none(),
15313                "retiring the old slot must not kill the replacement"
15314            );
15315            assert_eq!(
15316                group_count(),
15317                1,
15318                "only the live spawn's cgroup should remain"
15319            );
15320        }
15321        supervisor.begin_daemon_shutdown();
15322        let reap = tokio::spawn(async move {
15323            live.wait().await.unwrap();
15324        });
15325        supervisor
15326            .end_children_for_daemon_shutdown(false, std::future::pending())
15327            .await;
15328        reap.await.unwrap();
15329        assert_eq!(group_count(), 0);
15330        // A normal exit uses the same tree-cleanup path as a killed spawn.
15331        spec.program = PathBuf::from("/bin/true");
15332        spec.args.clear();
15333        let fresh_roster = ChildRoster::default();
15334        let mut short = spawn_child(
15335            &spec,
15336            None,
15337            None,
15338            &runtime.stderr_ring,
15339            None,
15340            &fresh_roster,
15341            Some(&placement),
15342        )
15343        .unwrap();
15344        short.wait().await.unwrap();
15345        assert_eq!(group_count(), 0);
15346        spec.module_id = "_".repeat(255);
15347        let mut long_id = spawn_child(
15348            &spec,
15349            None,
15350            None,
15351            &runtime.stderr_ring,
15352            None,
15353            &fresh_roster,
15354            Some(&placement),
15355        )
15356        .unwrap();
15357        long_id.wait().await.unwrap();
15358        assert_eq!(
15359            group_count(),
15360            0,
15361            "valid long module IDs must not exceed cgroup NAME_MAX"
15362        );
15363        fs::remove_dir(root.join("subc-modules")).unwrap();
15364        fs::remove_dir(root).unwrap();
15365        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15366    }
15367
15368    #[test]
15369    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15370        let path = Path::new("/definitely-missing-subc-cgroup");
15371        let mut command = Command::new("true");
15372        let error = apply_cgroup_placement(
15373            &mut command,
15374            &ModuleSpec {
15375                module_id: "broken-cgroup".to_string(),
15376                program: PathBuf::from("true"),
15377                args: Vec::new(),
15378                env: Vec::new(),
15379                reserved: false,
15380                reserved_prefixes: Vec::new(),
15381                protocol: ModuleProtocol::Subc,
15382                overlap: Default::default(),
15383            },
15384            path,
15385        )
15386        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15387        let reason = error.to_string();
15388
15389        assert!(
15390            matches!(error, SuperviseError::Cgroup { .. }),
15391            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15392        );
15393        assert!(
15394            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15395            "parent cgroup open failure must name cgroup.procs: {reason}"
15396        );
15397    }
15398
15399    #[tokio::test]
15400    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15401        let root = ScratchDir::new("supervisor-reap-cgroup");
15402        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15403        let placement = subc_cgroup::prepare_at(&root)
15404            .expect("prepare scratch cgroup root")
15405            .expect("scratch root has a cgroup.procs marker");
15406        let module_id = "reaped-module";
15407        let module = placement
15408            .module_path(module_id)
15409            .expect("create scratch module cgroup");
15410        let child = Command::new("true")
15411            .env("XDG_DATA_HOME", root.path())
15412            .env("XDG_RUNTIME_DIR", root.path())
15413            .env("XDG_CONFIG_HOME", root.path())
15414            .spawn()
15415            .expect("spawn short-lived child");
15416        let pid = child.id().expect("spawned child has pid");
15417        let mut child = SupervisedChild {
15418            child,
15419            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15420            protocol: ModuleProtocol::Subc,
15421            module_id: module_id.to_string(),
15422            cgroup_placement: Some(placement),
15423            stdout_pump: None,
15424            stderr_pump: None,
15425            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15426            spawned_at: tokio::time::Instant::now(),
15427            spawned_at_ms: 0,
15428            spawned_from: PathBuf::from("true"),
15429            launch_spec_digest: LaunchSpecDigest([0; 32]),
15430            spawned_file_identity: None,
15431            process_start_time: None,
15432            process_identity: None,
15433            pid,
15434            roster_guard: None,
15435            #[cfg(target_os = "macos")]
15436            privacy_exec: None,
15437            spawn_failure: None,
15438        };
15439
15440        child.wait().await.expect("reap short-lived child");
15441
15442        assert!(
15443            !module.exists(),
15444            "reaping the supervised child must remove its empty cgroup"
15445        );
15446    }
15447
15448    #[test]
15449    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15450        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15451        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15452        let placement = subc_cgroup::prepare_at(&root)
15453            .expect("prepare scratch cgroup root")
15454            .expect("scratch root has a cgroup.procs marker");
15455        let module = placement
15456            .module_path("surviving-module")
15457            .expect("create scratch module cgroup");
15458        fs::write(module.join("surviving-process"), b"still present")
15459            .expect("make scratch cgroup non-empty");
15460        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15461
15462        remove_module_cgroup(&placement, "surviving-module");
15463
15464        let logs = crate::router::test_log::captured_logs(&logs);
15465        assert!(
15466            module.exists(),
15467            "failed removal must leave the cgroup intact"
15468        );
15469        assert!(
15470            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15471                && logs.contains("surviving-module"),
15472            "best-effort removal must report the failure without returning it: {logs}"
15473        );
15474    }
15475
15476    #[test]
15477    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15478        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15479        let reason = SuperviseError::Spawn {
15480            program: PathBuf::from("/bin/true"),
15481            source: io::Error::from_raw_os_error(13),
15482            cgroup_path: Some(cgroup_path.clone()),
15483        }
15484        .to_string();
15485
15486        assert!(
15487            reason.contains(&cgroup_path.display().to_string()),
15488            "a pre_exec spawn failure must name the cgroup path: {reason}"
15489        );
15490    }
15491}
15492
15493#[cfg(test)]
15494mod spawn_subscriber_lag_tests {
15495    use super::*;
15496
15497    /// A subscriber whose connection stops draining is dropped once its frame
15498    /// channel fills. The client must learn that from a terminal Error frame
15499    /// after the frames already queued for it, not from a stream that simply
15500    /// goes quiet.
15501    #[tokio::test]
15502    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15503        let feed = SpawnEventFeed::default();
15504        feed.configure_incarnation("lag-incarnation".to_string());
15505        // A one-slot connection queue that nobody reads until the emits are
15506        // done: the forwarder parks on it and the subscriber channel fills.
15507        let (tx, mut rx) = mpsc::channel(1);
15508        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15509            .expect("subscribe");
15510        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15511        for index in 0..emitted {
15512            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15513            // Let the forwarder take what it can so the fill point is the
15514            // subscriber channel, not a scheduling accident.
15515            tokio::task::yield_now().await;
15516        }
15517        assert_eq!(
15518            feed.subscriber_count(),
15519            0,
15520            "the lagged subscriber must be removed"
15521        );
15522
15523        let mut data = Vec::new();
15524        let mut last = None;
15525        loop {
15526            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15527                .await
15528                .expect("the forwarder must finish once the subscriber is dropped");
15529            let Some(outbound) = next else { break };
15530            let frame = outbound.frame;
15531            if frame.header.ty == FrameType::StreamData {
15532                assert!(last.is_none(), "no data may follow the terminal frame");
15533                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15534                data.push(event.cursor.seq);
15535            } else {
15536                assert!(last.is_none(), "exactly one terminal frame");
15537                last = Some(frame);
15538            }
15539        }
15540        assert!(!data.is_empty(), "queued frames drain before the terminal");
15541        for pair in data.windows(2) {
15542            assert_eq!(
15543                pair[1],
15544                pair[0] + 1,
15545                "queued frames arrive dense and in order"
15546            );
15547        }
15548        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15549        assert_eq!(terminal.header.ty, FrameType::Error);
15550        assert_eq!(terminal.header.corr, 7);
15551        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15552        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15553        let detail = body.detail.expect("lagged error carries detail");
15554        assert_eq!(
15555            detail["first_undelivered_cursor"]["seq"],
15556            data.last().unwrap() + 1,
15557            "the named cursor is the first event the subscriber did not receive"
15558        );
15559        assert_eq!(
15560            detail["first_undelivered_cursor"]["daemon_incarnation"],
15561            "lag-incarnation"
15562        );
15563    }
15564}
15565
15566#[cfg(test)]
15567mod terminal_history_read_concurrency_tests {
15568    use super::*;
15569    use crate::terminal_journal::read_pause;
15570    use cortexkit_test_support::ScratchDir;
15571    use std::sync::mpsc as std_mpsc;
15572
15573    fn journaled_ring(
15574        journal: &Arc<crate::terminal_journal::TerminalJournal>,
15575    ) -> Arc<Mutex<TerminalRing>> {
15576        Arc::new(Mutex::new(
15577            TerminalRing::new(TerminalRingConfig::default(), 1)
15578                .with_journal(Some(Arc::clone(journal))),
15579        ))
15580    }
15581
15582    fn crash(at_ms: u64) -> ExitReport {
15583        ExitReport {
15584            kind: ExitKind::Crash,
15585            code: Some(1),
15586            signal: None,
15587            at_ms,
15588        }
15589    }
15590
15591    /// Record an exit on another thread and report whether it finished within
15592    /// `bound`. The recorder thread is left running if it did not.
15593    fn record_within(
15594        module_id: &'static str,
15595        ring: &Arc<Mutex<TerminalRing>>,
15596        at_ms: u64,
15597        bound: Duration,
15598    ) -> bool {
15599        let ring = Arc::clone(ring);
15600        let (done, done_rx) = std_mpsc::channel();
15601        std::thread::spawn(move || {
15602            record_terminal(
15603                module_id,
15604                &ring,
15605                &SpawnEventFeed::default(),
15606                &crash(at_ms),
15607                TerminalDisposition::Restarting,
15608            );
15609            let _ = done.send(());
15610        });
15611        done_rx.recv_timeout(bound).is_ok()
15612    }
15613
15614    /// A history read in progress must not hold the journal writer (which every
15615    /// module's exit recording needs) or the module's own ring. Exits recorded
15616    /// while the read is paused complete promptly; the paused read answers as of
15617    /// the moment it started, and the next read has each exit exactly once.
15618    #[test]
15619    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15620        let dir = ScratchDir::new("terminal-history-concurrent-read");
15621        let path = dir.join("terminals.jsonl");
15622        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15623            path.clone(),
15624            "daemon".into(),
15625        ));
15626        let reader_ring = journaled_ring(&journal);
15627        let other_ring = journaled_ring(&journal);
15628        assert!(record_within(
15629            "reader-module",
15630            &reader_ring,
15631            10,
15632            Duration::from_secs(5)
15633        ));
15634
15635        let (started, release) = read_pause::install(&path);
15636        let reading = {
15637            let ring = Arc::clone(&reader_ring);
15638            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15639        };
15640        started
15641            .recv_timeout(Duration::from_secs(5))
15642            .expect("the history read reached its pause");
15643
15644        let bound = Duration::from_secs(1);
15645        assert!(
15646            record_within("other-module", &other_ring, 20, bound),
15647            "another module's exit waited on a history read (journal writer held)"
15648        );
15649        assert!(
15650            record_within("reader-module", &reader_ring, 30, bound),
15651            "the read module's own exit waited on its history read (ring held)"
15652        );
15653
15654        drop(release);
15655        let paused = reading.join().unwrap();
15656        assert_eq!(
15657            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15658            vec![10],
15659            "an exit recorded after the read began lands in neither half of it"
15660        );
15661        assert_eq!(paused.journal_skipped_lines, 0);
15662        assert_eq!(paused.journal_read_errors, 0);
15663
15664        let after = durable_terminal_history_of(&reader_ring, "reader-module");
15665        assert_eq!(
15666            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15667            vec![10, 30],
15668            "the next read merges ring and journal with no duplicate"
15669        );
15670        assert_eq!(after.journal_skipped_lines, 0);
15671    }
15672}
15673
15674/// What a restart does with the exited process's output readers. These drive
15675/// the same settle functions the supervisor calls, with a reader the test
15676/// holds, so a reader that has not been scheduled by the bound is a controlled
15677/// input rather than something only a loaded machine produces.
15678#[cfg(test)]
15679mod stderr_settle_tests {
15680    use std::{
15681        future::Future,
15682        io,
15683        pin::Pin,
15684        sync::{Arc, Mutex},
15685        task::{Context, Poll},
15686        time::Duration,
15687    };
15688
15689    use tokio::{
15690        io::{AsyncRead, ReadBuf},
15691        sync::oneshot,
15692        time::Instant,
15693    };
15694
15695    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15696    use crate::stderr_tail::{
15697        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15698        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15699    };
15700
15701    const BOUND: Duration = Duration::from_millis(250);
15702
15703    /// Yields `before`, then stays pending until the gate is released, then
15704    /// yields `after` and reaches EOF. The bytes after the gate were written
15705    /// by a process that has already exited; only the reader is behind.
15706    struct HeldReader {
15707        before: Option<Vec<u8>>,
15708        gate: Option<oneshot::Receiver<()>>,
15709        after: io::Cursor<Vec<u8>>,
15710    }
15711
15712    impl AsyncRead for HeldReader {
15713        fn poll_read(
15714            mut self: Pin<&mut Self>,
15715            cx: &mut Context<'_>,
15716            buf: &mut ReadBuf<'_>,
15717        ) -> Poll<io::Result<()>> {
15718            if let Some(bytes) = self.before.take() {
15719                buf.put_slice(&bytes);
15720                return Poll::Ready(Ok(()));
15721            }
15722            if let Some(gate) = self.gate.as_mut() {
15723                match Pin::new(gate).poll(cx) {
15724                    Poll::Pending => return Poll::Pending,
15725                    Poll::Ready(_) => self.gate = None,
15726                }
15727            }
15728            Pin::new(&mut self.after).poll_read(cx, buf)
15729        }
15730    }
15731
15732    struct DiscardSink;
15733
15734    impl OutputSink for DiscardSink {
15735        fn write_line(&mut self, _line: &[u8]) {}
15736    }
15737
15738    fn line(text: &str) -> TailEntry {
15739        TailEntry::Line {
15740            text: text.to_string(),
15741            truncated: false,
15742            at_ms: None,
15743        }
15744    }
15745
15746    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15747        ring.lock().unwrap()
15748    }
15749
15750    /// Start a reader for a new process generation that delivers `before`
15751    /// immediately and `after` only once the returned sender fires (or is
15752    /// dropped).
15753    fn held_pump(
15754        ring: &Arc<Mutex<StderrRing>>,
15755        before: &str,
15756        after: &str,
15757    ) -> (StderrPump, oneshot::Sender<()>) {
15758        let generation = lock(ring).begin_process();
15759        let (release, gate) = oneshot::channel();
15760        let reader = HeldReader {
15761            before: Some(before.as_bytes().to_vec()),
15762            gate: Some(gate),
15763            after: io::Cursor::new(after.as_bytes().to_vec()),
15764        };
15765        let task = tokio::spawn(pump_stderr_to(
15766            reader,
15767            Arc::clone(ring),
15768            generation,
15769            DiscardSink,
15770        ));
15771        (StderrPump { task, generation }, release)
15772    }
15773
15774    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15775        for _ in 0..1000 {
15776            if done(&lock(ring)) {
15777                return;
15778            }
15779            tokio::time::sleep(Duration::from_millis(1)).await;
15780        }
15781        panic!(
15782            "ring never reached the expected state: {:?}",
15783            lock(ring).snapshot(None, None)
15784        );
15785    }
15786
15787    #[tokio::test(start_paused = true)]
15788    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15789        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15790        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15791
15792        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15793        let before_release = lock(&ring).snapshot(None, None);
15794        assert!(
15795            matches!(before_release.capture, CaptureState::Incomplete { .. }),
15796            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15797        );
15798
15799        // The restart: the next process starts and writes before the old
15800        // reader catches up.
15801        let next = lock(&ring).begin_process();
15802        lock(&ring).push_line_from(next, "next process booting");
15803        release.send(()).unwrap();
15804        wait_until(&ring, |ring| {
15805            ring.snapshot(None, None).capture == CaptureState::Captured
15806        })
15807        .await;
15808
15809        assert_eq!(
15810            untimed(lock(&ring).snapshot(None, None).entries),
15811            vec![
15812                line("booting"),
15813                line("config error: missing storage"),
15814                TailEntry::ProcessStart,
15815                line("next process booting"),
15816            ],
15817            "the crash's last line must survive a slow reader and stay in the crashed process's section"
15818        );
15819    }
15820
15821    #[tokio::test(start_paused = true)]
15822    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15823    ) {
15824        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15825        // `_held` is never fired: a descendant keeps the pipe open for the
15826        // whole test.
15827        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15828
15829        let started = Instant::now();
15830        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15831        assert_eq!(
15832            started.elapsed(),
15833            BOUND,
15834            "the restart must wait exactly the bound for a pipe that stays open, no longer"
15835        );
15836
15837        let next = lock(&ring).begin_process();
15838        lock(&ring).push_line_from(next, "next process booting");
15839        tokio::time::sleep(Duration::from_secs(60)).await;
15840
15841        let snapshot = lock(&ring).snapshot(None, None);
15842        match &snapshot.capture {
15843            CaptureState::Incomplete { reason } => assert!(
15844                reason.contains("had not reached EOF") && reason.contains("250ms"),
15845                "the reason must say what is missing and after how long: {reason}"
15846            ),
15847            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15848        }
15849        assert_eq!(
15850            untimed(snapshot.entries),
15851            vec![
15852                line("parent exiting"),
15853                TailEntry::ProcessStart,
15854                line("next process booting"),
15855            ]
15856        );
15857    }
15858
15859    #[tokio::test(start_paused = true)]
15860    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15861        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15862        let (pump, release) = held_pump(&ring, "one\n", "two\n");
15863        release.send(()).unwrap();
15864
15865        settle_stderr_pump("clean", &ring, pump, BOUND).await;
15866
15867        let snapshot = lock(&ring).snapshot(None, None);
15868        assert_eq!(snapshot.capture, CaptureState::Captured);
15869        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15870    }
15871
15872    #[tokio::test(start_paused = true)]
15873    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15874        const BURST: usize = 150;
15875        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15876        let path = capture.join("two-pipe.stderr.log");
15877        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15878        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15879        let generation = lock(&ring).begin_process();
15880        let stdout = (0..BURST)
15881            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15882            .collect::<String>();
15883        let stderr = (0..BURST)
15884            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15885            .collect::<String>();
15886        // Model a child that has written both bursts and exited. Hold the stdout
15887        // reader after one line and a partial prefix, as if pipe readiness or
15888        // capture-file writes had delayed it. The final line also needs EOF to
15889        // flush: neither complete nor partial buffered output may be discarded.
15890        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15891        let (release, gate) = oneshot::channel();
15892        let reader = HeldReader {
15893            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15894            gate: Some(gate),
15895            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15896        };
15897        let (finished, completion) = oneshot::channel();
15898        let stdout_sink = sink.clone();
15899        let pump = tokio::spawn(async move {
15900            pump_stdout_to(reader, stdout_sink).await;
15901            finished.send(()).unwrap();
15902        });
15903        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15904
15905        let started = Instant::now();
15906        settle_stdout_pump("two-pipe", pump, BOUND).await;
15907        assert_eq!(
15908            started.elapsed(),
15909            BOUND,
15910            "a late reader must not hold up restart"
15911        );
15912        assert_eq!(
15913            std::fs::read_to_string(&path).unwrap().lines().count(),
15914            BURST + 1
15915        );
15916
15917        release
15918            .send(())
15919            .expect("stdout reader must survive the exit drain bound");
15920        tokio::time::timeout(Duration::from_secs(1), completion)
15921            .await
15922            .expect("released stdout reader must reach EOF")
15923            .expect("stdout capture must finish at EOF, not be cancelled");
15924        let contents = std::fs::read_to_string(&path).unwrap();
15925        let mut lines = contents
15926            .lines()
15927            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15928            .collect::<Vec<_>>();
15929        let mut expected = (0..BURST)
15930            .flat_map(|i| {
15931                [
15932                    format!("out-{i:04}-{}", "o".repeat(64)),
15933                    format!("err-{i:04}-{}", "e".repeat(64)),
15934                ]
15935            })
15936            .collect::<Vec<_>>();
15937        lines.sort_unstable();
15938        expected.sort_unstable();
15939        assert_eq!(
15940            lines, expected,
15941            "both pipes must retain every complete line exactly once"
15942        );
15943    }
15944}
15945
15946/// Containment of a module's process tree (issue #109).
15947///
15948/// The behaviour these defend against is a module helper surviving its module:
15949/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
15950/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
15951/// compounds it.
15952///
15953/// They run against the SUPERVISOR rather than the job-object crate because the
15954/// claim is about teardown: a crate-level test proves a job can reap a tree, not
15955/// that the daemon's drain path reaches it.
15956///
15957/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
15958/// lane there is a separate containment path with its own tests.
15959#[cfg(all(test, windows))]
15960mod job_containment_tests {
15961    use super::*;
15962    use cortexkit_test_support::ScratchDir;
15963    use std::{
15964        path::{Path, PathBuf},
15965        sync::{Arc, Mutex},
15966        time::{Duration, Instant},
15967    };
15968
15969    /// The stub, expected beside this test executable.
15970    ///
15971    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
15972    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
15973    /// failure then reads as a broken test rather than an unbuilt dependency.
15974    fn stub_path() -> PathBuf {
15975        let mut path = std::env::current_exe().expect("current_exe available in tests");
15976        path.pop();
15977        path.pop();
15978        path.push("fake-aft-stub.exe");
15979        assert!(
15980            path.exists(),
15981            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15982             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15983            path.display()
15984        );
15985        path
15986    }
15987
15988    /// Poll for the grandchild pid the stub records, and parse it.
15989    fn read_grandchild_pid(path: &Path) -> u32 {
15990        let deadline = Instant::now() + Duration::from_secs(10);
15991        loop {
15992            if let Ok(contents) = std::fs::read_to_string(path) {
15993                if let Ok(pid) = contents.trim().parse() {
15994                    return pid;
15995                }
15996            }
15997            assert!(
15998                Instant::now() < deadline,
15999                "the stub never recorded a grandchild pid at {}",
16000                path.display()
16001            );
16002            std::thread::sleep(Duration::from_millis(10));
16003        }
16004    }
16005
16006    /// Everything one fixture run needs, so the two tests below differ in exactly
16007    /// one place: whether the child is contained.
16008    struct Fixture {
16009        _dir: ScratchDir,
16010        module_id: String,
16011        grandchild: u32,
16012        child: Option<SupervisedChild>,
16013        registry: Arc<Registry>,
16014        snapshot: Arc<Mutex<SupervisorSnapshot>>,
16015        terminal_ring: Arc<Mutex<TerminalRing>>,
16016        spawn_events: SpawnEventFeed,
16017    }
16018
16019    fn fixture(label: &str, module_id: &str) -> Fixture {
16020        let dir = ScratchDir::new(label);
16021        let pid_file = dir.join("grandchild.pid");
16022        let supervisor = Supervisor::new_for_test(
16023            Arc::new(Registry::default()),
16024            RestartPolicy::new(3, Duration::ZERO),
16025        );
16026        let runtime = supervisor.runtime_config();
16027        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16028        let spec = ModuleSpec {
16029            module_id: module_id.to_string(),
16030            program: stub_path(),
16031            // Zero args deliberately: a `--subc` argument would make the stub dial
16032            // a daemon that is not there, and the failure would land in the same
16033            // stderr ring this fixture exists to keep quiet.
16034            args: Vec::new(),
16035            env: vec![
16036                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16037                (
16038                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16039                    pid_file.display().to_string(),
16040                ),
16041            ],
16042            reserved: false,
16043            reserved_prefixes: Vec::new(),
16044            protocol: ModuleProtocol::Subc,
16045            overlap: Default::default(),
16046        };
16047        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16048            .expect("spawn the supervised fixture");
16049        let grandchild = read_grandchild_pid(&pid_file);
16050        Fixture {
16051            _dir: dir,
16052            module_id: module_id.to_string(),
16053            grandchild,
16054            child: Some(child),
16055            registry: Arc::new(Registry::default()),
16056            snapshot,
16057            terminal_ring: Arc::clone(&runtime.terminal_ring),
16058            spawn_events: SpawnEventFeed::default(),
16059        }
16060    }
16061
16062    impl Fixture {
16063        /// Drain through the supervisor's own teardown path.
16064        async fn drain(&mut self) {
16065            let child = self
16066                .child
16067                .take()
16068                .expect("the fixture child is still present");
16069            drain_child_to_state(
16070                &self.module_id,
16071                ModuleProtocol::Subc,
16072                // No forwarding table in this fixture, so nothing reaches the
16073                // child over a connection.
16074                StopNotice::NotSent,
16075                &self.registry,
16076                None,
16077                &self.snapshot,
16078                &self.terminal_ring,
16079                &self.spawn_events,
16080                child,
16081                Duration::from_millis(500),
16082                ModuleState::Stopped,
16083                Some(false),
16084            )
16085            .await
16086            .expect("drain the supervised fixture");
16087        }
16088    }
16089
16090    /// Teardown reaps the grandchild, not merely the direct child.
16091    ///
16092    /// This is the assertion the change exists for. Before containment the
16093    /// grandchild survived: it is a separate process, and `start_kill` is
16094    /// `TerminateProcess` scoped to one pid.
16095    #[tokio::test]
16096    async fn teardown_reaps_the_grandchild() {
16097        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16098        let grandchild = fixture.grandchild;
16099
16100        assert!(
16101            subc_jobobject::process_exists(grandchild),
16102            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16103        );
16104
16105        fixture.drain().await;
16106
16107        assert!(
16108            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16109            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16110        );
16111    }
16112
16113    /// The mutation control: with containment withheld, the grandchild survives
16114    /// the same kill.
16115    ///
16116    /// This is the defect reproduction from #109 — a direct-child kill reaches
16117    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16118    /// supervisor because `spawn_and_mark_running` now always contains on
16119    /// Windows, which is the point: there is no longer a path that spawns
16120    /// uncontained, so the control has to construct one.
16121    ///
16122    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16123    /// grandchild ever dies here, that test is passing for a reason unrelated to
16124    /// the job object and the containment claim is unproven.
16125    #[test]
16126    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16127        let dir = ScratchDir::new("teardown-uncontained");
16128        let pid_file = dir.join("grandchild.pid");
16129        let mut child = std::process::Command::new(stub_path())
16130            .env("FAKE_AFT_NEVER_CONNECT", "1")
16131            .env(
16132                "FAKE_AFT_GRANDCHILD_PID_FILE",
16133                pid_file.display().to_string(),
16134            )
16135            .stdin(std::process::Stdio::null())
16136            .stdout(std::process::Stdio::null())
16137            .stderr(std::process::Stdio::null())
16138            .spawn()
16139            .expect("spawn the uncontained fixture");
16140        let grandchild = read_grandchild_pid(&pid_file);
16141
16142        // Exactly what the pre-fix teardown did: kill the direct child.
16143        child.kill().expect("kill the direct child");
16144        let _ = child.wait();
16145
16146        assert!(
16147            subc_jobobject::process_exists(grandchild),
16148            "grandchild {grandchild} died with the direct child, so this control no longer \
16149             distinguishes contained from uncontained teardown and the regression test is \
16150             passing vacuously"
16151        );
16152
16153        // The orphan this control demonstrates is the leak the fix prevents, so
16154        // the control must not leave one behind.
16155        kill_tree(grandchild);
16156    }
16157
16158    /// Crash durability: closing the containment handle reaps the tree with no
16159    /// teardown code running at all.
16160    ///
16161    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16162    /// call anything — and it is why containment is a kernel property of the
16163    /// handle rather than a step in the drain. Discovered by getting the
16164    /// mutation control wrong: clearing `job` to "disable" containment instead
16165    /// killed the tree, which is the guarantee, not a mistake.
16166    #[tokio::test]
16167    async fn dropping_containment_reaps_the_grandchild() {
16168        let mut fixture = fixture("drop-containment", "tree-drop");
16169        let grandchild = fixture.grandchild;
16170
16171        assert!(subc_jobobject::process_exists(grandchild));
16172
16173        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16174        fixture.child.as_mut().expect("child present").job = None;
16175
16176        assert!(
16177            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16178            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16179             crash would leave the tree behind"
16180        );
16181    }
16182
16183    /// Kill a pid and its tree, then confirm it is gone.
16184    fn kill_tree(pid: u32) {
16185        let _ = std::process::Command::new("taskkill.exe")
16186            .args(["/PID", &pid.to_string(), "/T", "/F"])
16187            .stdin(std::process::Stdio::null())
16188            .stdout(std::process::Stdio::null())
16189            .stderr(std::process::Stdio::null())
16190            .status();
16191        assert!(
16192            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16193            "could not clean up grandchild {pid}"
16194        );
16195    }
16196}
16197
16198#[cfg(test)]
16199mod privacy_trampoline_configuration_tests {
16200    #[tokio::test]
16201    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16202    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16203        #[cfg(target_os = "macos")]
16204        {
16205            let supervisor = super::Supervisor::new(
16206                std::sync::Arc::new(crate::Registry::default()),
16207                super::RestartPolicy::default(),
16208            );
16209            let error = supervisor.spawn(spec()).unwrap_err();
16210            assert!(
16211                error
16212                    .to_string()
16213                    .contains("no privacy trampoline configured"),
16214                "{error}"
16215            );
16216        }
16217    }
16218
16219    #[tokio::test]
16220    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16221    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16222        #[cfg(target_os = "macos")]
16223        {
16224            let supervisor = super::Supervisor::new(
16225                std::sync::Arc::new(crate::Registry::default()),
16226                super::RestartPolicy::default(),
16227            )
16228            .with_privacy_trampoline(std::env::current_exe().unwrap());
16229            let error = supervisor.spawn(spec()).unwrap_err();
16230            assert!(
16231                error
16232                    .to_string()
16233                    .contains("binary does not implement the privacy trampoline protocol"),
16234                "{error}"
16235            );
16236        }
16237    }
16238
16239    #[cfg(target_os = "macos")]
16240    fn spec() -> super::ModuleSpec {
16241        super::ModuleSpec {
16242            module_id: "privacy-configuration".into(),
16243            program: "/bin/sleep".into(),
16244            args: vec!["30".into()],
16245            env: vec![],
16246            reserved: false,
16247            reserved_prefixes: vec![],
16248            protocol: subc_control::ModuleProtocol::None,
16249            overlap: super::ModuleOverlap::Exclusive,
16250        }
16251    }
16252}
16253
16254#[cfg(test)]
16255mod privacy_exec_boundary_tests {
16256    #[cfg(target_os = "macos")]
16257    use super::*;
16258    #[cfg(target_os = "macos")]
16259    use std::{
16260        io::{Read, Write},
16261        net::{TcpListener, TcpStream},
16262    };
16263
16264    /// Unit-test-only pause at the actual early image read, not at a later
16265    /// status read. Production supervisors never inspect this environment key.
16266    #[cfg(target_os = "macos")]
16267    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16268        if let Some((_, path)) = spec
16269            .env
16270            .iter()
16271            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16272        {
16273            let mut barrier = TcpStream::connect(path).unwrap();
16274            barrier
16275                .set_read_timeout(Some(Duration::from_secs(30)))
16276                .unwrap();
16277            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16278            let mut release = [0];
16279            barrier.read_exact(&mut release).unwrap();
16280            assert_eq!(&release, b"X");
16281        }
16282    }
16283
16284    #[cfg(target_os = "macos")]
16285    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16286        ModuleSpec {
16287            module_id: "privacy-boundary".into(),
16288            program: program.into(),
16289            args: args.iter().map(|arg| (*arg).into()).collect(),
16290            env: vec![],
16291            reserved: false,
16292            reserved_prefixes: vec![],
16293            protocol: ModuleProtocol::None,
16294            overlap: ModuleOverlap::Exclusive,
16295        }
16296    }
16297
16298    #[cfg(target_os = "macos")]
16299    async fn accept(listener: TcpListener) -> TcpStream {
16300        // Socket readiness, not elapsed time, establishes both pause points.
16301        let listener = tokio::net::TcpListener::from_std({
16302            listener.set_nonblocking(true).unwrap();
16303            listener
16304        })
16305        .unwrap();
16306        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16307            .await
16308            .unwrap()
16309            .unwrap();
16310        let stream = stream.into_std().unwrap();
16311        stream.set_nonblocking(false).unwrap();
16312        stream
16313            .set_read_timeout(Some(Duration::from_secs(30)))
16314            .unwrap();
16315        stream
16316    }
16317
16318    #[tokio::test]
16319    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16320    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16321        #[cfg(target_os = "macos")]
16322        {
16323            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16324            // Loopback sockets also work when the replay adapter's TMPDIR is
16325            // longer than Darwin's Unix-domain socket path limit.
16326            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16327            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16328            let record = root.join("live-children.json");
16329            let supervisor =
16330                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16331                    .with_live_children_record(&record);
16332            let runtime = supervisor.runtime_config();
16333            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16334            let mut spec = spec("/bin/sleep", &["30"]);
16335            spec.env = vec![
16336                (
16337                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16338                    exec_listener.local_addr().unwrap().to_string(),
16339                ),
16340                (
16341                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16342                    sample_listener.local_addr().unwrap().to_string(),
16343                ),
16344            ];
16345            let spawn = tokio::task::spawn_blocking(move || {
16346                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16347            });
16348            let mut sample = accept(sample_listener).await;
16349            let mut pid = [0; 4];
16350            sample.read_exact(&mut pid).unwrap();
16351            let pid = u32::from_ne_bytes(pid);
16352            let mut exec = accept(exec_listener).await;
16353            let mut ready = [0];
16354            exec.read_exact(&mut ready).unwrap();
16355            assert_eq!(&ready, b"R");
16356            // The early read is guaranteed to see a real, non-null trampoline
16357            // image: the fixture has reached its barrier and cannot exec yet.
16358            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16359            assert_eq!(
16360                observe_spawned_image(pid).unwrap().executable,
16361                Some(trampoline)
16362            );
16363            sample.write_all(b"X").unwrap();
16364            let mut child = spawn.await.unwrap();
16365            let early = crate::live_children::read_record(&record).unwrap();
16366            assert_eq!(early.len(), 1);
16367            assert_eq!(early[0].pid, pid);
16368            assert_eq!(
16369                early[0].executable, None,
16370                "unconfirmed trampoline image entered the roster"
16371            );
16372            assert!(child.report_ready.get().is_none());
16373            // The barrier's duration is unrelated to the production five-second
16374            // exec budget. Start the test's confirmation budget upon release.
16375            child.privacy_exec.as_mut().unwrap().deadline =
16376                tokio::time::Instant::now() + Duration::from_secs(30);
16377            exec.write_all(b"X").unwrap();
16378            child.confirm_privacy_exec().await;
16379            assert_eq!(child.spawn_failure, None);
16380            assert!(child.report_ready.get().is_some());
16381            let confirmed = crate::live_children::read_record(&record).unwrap();
16382            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16383            assert_ne!(module, trampoline);
16384            assert_eq!(confirmed[0].executable, Some(module.into()));
16385            child.start_kill().unwrap();
16386            child.wait().await.unwrap();
16387            child.release_roster();
16388        }
16389    }
16390
16391    #[tokio::test]
16392    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16393    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16394        #[cfg(target_os = "macos")]
16395        {
16396            let registry = Arc::new(Registry::default());
16397            let policy = RestartPolicy::new(0, Duration::ZERO);
16398            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16399            let runtime = supervisor.runtime_config();
16400            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16401            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16402            // Drive spawn and confirmation separately instead of starting the
16403            // monitor. WNOWAIT observes a real exit without consuming its status,
16404            // so confirmation's first try_wait must take the already-exited arm.
16405            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16406            let pid = child.pid;
16407            tokio::task::spawn_blocking(move || {
16408                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16409            })
16410            .await
16411            .unwrap()
16412            .unwrap();
16413            child.privacy_exec.as_mut().unwrap().deadline =
16414                tokio::time::Instant::now() + Duration::from_secs(30);
16415            let status = child.wait().await.unwrap();
16416            assert_eq!(status.code(), Some(121));
16417            assert!(child.privacy_exec.is_none());
16418            assert!(
16419                child.report_ready.get().is_none(),
16420                "an exited module must not publish a live pid"
16421            );
16422            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16423            on_child_exit(
16424                &spec,
16425                policy,
16426                &registry,
16427                &snapshot,
16428                &runtime.terminal_ring,
16429                &runtime.spawn_events,
16430                &runtime.child_roster,
16431                report,
16432            )
16433            .await;
16434            let state = lock_snapshot(&snapshot).unwrap();
16435            assert_eq!(state.state, ModuleState::Failed);
16436            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16437            assert_eq!(state.reported_pid(), None);
16438            drop(state);
16439            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16440            assert_eq!(history.entries.len(), 1);
16441            let terminal = &history.entries[0];
16442            assert_eq!(terminal.exit_code, Some(121));
16443            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16444            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16445            assert_eq!(
16446                terminal.disposition_detail.as_deref(),
16447                Some(policy.budget_exhausted_detail().as_str()),
16448                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16449            );
16450            assert_eq!(child.spawn_failure, None);
16451            child.release_roster();
16452        }
16453    }
16454}
16455
16456/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16457/// descriptor 3, without an environment copy. The shell records the nonce
16458/// and its environment after exec so these tests observe the real handover.
16459#[cfg(all(test, unix))]
16460mod launch_nonce_descriptor_tests {
16461    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16462    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16463    use cortexkit_test_support::ScratchDir;
16464    use std::{
16465        path::PathBuf,
16466        sync::{Arc, Mutex},
16467        time::{Duration, Instant},
16468    };
16469
16470    async fn probe(role: super::SpawnRole) {
16471        let scratch = ScratchDir::new("launch-nonce-descriptor");
16472        let fd_copy = scratch.join("from-descriptor");
16473        let env_copy = scratch.join("environment");
16474        let script = format!(
16475            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16476            fd = fd_copy.display(), env = env_copy.display(),
16477        );
16478        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16479        let spec = ModuleSpec {
16480            module_id: "nonce-descriptor-probe".to_string(),
16481            program: PathBuf::from("/bin/sh"),
16482            args: vec!["-c".to_string(), script],
16483            env: vec![
16484                xdg("XDG_DATA_HOME"),
16485                xdg("XDG_RUNTIME_DIR"),
16486                xdg("XDG_CONFIG_HOME"),
16487                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16488            ],
16489            reserved: true,
16490            reserved_prefixes: Vec::new(),
16491            protocol: ModuleProtocol::Subc,
16492            overlap: Default::default(),
16493        };
16494        let handle = SupervisorHandle::new();
16495        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16496        let roster = ChildRoster::default();
16497        #[cfg(target_os = "macos")]
16498        {
16499            let path = super::test_privacy_trampoline();
16500            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16501        }
16502        let child = super::spawn_child_in_slot(
16503            &spec,
16504            None,
16505            Some(&handle),
16506            &ring,
16507            None,
16508            &roster,
16509            #[cfg(target_os = "linux")]
16510            None,
16511            role,
16512            matches!(role, super::SpawnRole::SwapCandidate),
16513        )
16514        .expect("spawn probe");
16515        let deadline = Instant::now() + Duration::from_secs(10);
16516        while !(fd_copy.exists() && env_copy.exists()) {
16517            assert!(Instant::now() < deadline, "probe never wrote its copies");
16518            tokio::time::sleep(Duration::from_millis(20)).await;
16519        }
16520        let nonce = std::fs::read_to_string(fd_copy).unwrap();
16521        assert!(!nonce.is_empty());
16522        let environment = std::fs::read_to_string(env_copy).unwrap();
16523        assert!(environment
16524            .lines()
16525            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16526        let copy = environment
16527            .lines()
16528            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16529        assert_eq!(
16530            copy, None,
16531            "Unix children must never receive the environment nonce"
16532        );
16533        if matches!(role, super::SpawnRole::Plain) {
16534            assert_eq!(
16535                handle.spawn_nonce(&spec.module_id).as_deref(),
16536                Some(nonce.as_str())
16537            );
16538        }
16539        drop(child);
16540    }
16541
16542    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16543    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16544        probe(super::SpawnRole::Plain).await;
16545    }
16546
16547    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16548    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16549        probe(super::SpawnRole::SwapCandidate).await;
16550    }
16551}
16552
16553#[cfg(all(test, target_os = "linux"))]
16554mod cgroup_containment_tests {
16555    use super::*;
16556    use cortexkit_test_support::ScratchDir;
16557
16558    fn running(pid: u32) -> bool {
16559        // An orphan can remain a zombie until the container init reaps it.
16560        std::fs::read_to_string(format!("/proc/{pid}/stat"))
16561            .ok()
16562            .and_then(|stat| {
16563                stat.rsplit_once(") ")
16564                    .map(|(_, rest)| rest.starts_with('Z'))
16565            })
16566            .is_some_and(|zombie| !zombie)
16567    }
16568
16569    #[tokio::test]
16570    async fn linux_teardown_reaps_the_grandchild() {
16571        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16572    }
16573
16574    #[tokio::test]
16575    async fn linux_shutdown_straggler_reaps_the_grandchild() {
16576        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16577    }
16578
16579    async fn teardown_tree(test_name: &str, shutdown: bool) {
16580        let dir = ScratchDir::new(test_name);
16581        let root = PathBuf::from(format!(
16582            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16583            std::process::id(),
16584            unix_ms_now()
16585        ));
16586        if let Err(error) = std::fs::create_dir(&root) {
16587            assert!(
16588                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16589                "required cgroup test cannot execute: {error}"
16590            );
16591            eprintln!(
16592                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16593                root.display()
16594            );
16595            return;
16596        }
16597        let placement = subc_cgroup::prepare_at(&root)
16598            .expect("prepare isolated kernel cgroup")
16599            .expect("isolated cgroup is delegated");
16600        let module_id = "tree-teardown";
16601        let module = placement
16602            .module_path(module_id)
16603            .expect("create isolated module cgroup");
16604        if !module.join("cgroup.kill").exists() {
16605            std::fs::remove_dir(&module).unwrap();
16606            std::fs::remove_dir(root.join("subc-modules")).unwrap();
16607            std::fs::remove_dir(&root).unwrap();
16608            assert!(
16609                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16610                "required cgroup.kill interface unavailable"
16611            );
16612            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16613            return;
16614        }
16615        let supervisor = Supervisor::new_for_test(
16616            Arc::new(Registry::default()),
16617            RestartPolicy::new(3, Duration::ZERO),
16618        )
16619        .with_cgroup_placement(Some(placement));
16620        let mut runtime = supervisor.runtime_config();
16621        runtime.child_roster = runtime
16622            .child_roster
16623            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16624        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16625        let pid_file = dir.join("grandchild.pid");
16626        let spec = ModuleSpec {
16627            module_id: module_id.to_string(),
16628            program: PathBuf::from("/bin/sh"),
16629            args: vec![
16630                "-c".into(),
16631                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16632                "fixture".into(),
16633                pid_file.display().to_string(),
16634            ],
16635            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16636                .into_iter()
16637                .map(|key| (key.to_string(), dir.display().to_string()))
16638                .collect(),
16639            reserved: false,
16640            reserved_prefixes: Vec::new(),
16641            protocol: ModuleProtocol::None,
16642            overlap: Default::default(),
16643        };
16644        let child =
16645            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16646        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16647        let grandchild: u32 = loop {
16648            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16649                if let Ok(pid) = contents.trim().parse() {
16650                    break pid;
16651                }
16652            }
16653            assert!(
16654                tokio::time::Instant::now() < deadline,
16655                "grandchild pid was not recorded"
16656            );
16657            tokio::time::sleep(Duration::from_millis(10)).await;
16658        };
16659        assert!(
16660            running(grandchild),
16661            "grandchild must be alive before teardown"
16662        );
16663        if shutdown {
16664            let mut child = child;
16665            crate::child_roster::end_children_for_daemon_shutdown(
16666                &runtime.child_roster,
16667                false,
16668                std::future::pending(),
16669            )
16670            .await;
16671            child.wait().await.expect("reap shutdown straggler");
16672        } else {
16673            drain_child_to_state(
16674                module_id,
16675                ModuleProtocol::None,
16676                StopNotice::NotSent,
16677                &Registry::default(),
16678                None,
16679                &snapshot,
16680                &runtime.terminal_ring,
16681                &SpawnEventFeed::default(),
16682                child,
16683                Duration::from_millis(100),
16684                ModuleState::Stopped,
16685                Some(false),
16686            )
16687            .await
16688            .expect("real supervisor teardown");
16689        }
16690        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16691        while running(grandchild) && tokio::time::Instant::now() < deadline {
16692            tokio::time::sleep(Duration::from_millis(10)).await;
16693        }
16694        let survived = running(grandchild);
16695        // Kill a surviving grandchild so a failed test does not leave it behind.
16696        if survived {
16697            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16698            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16699            tokio::time::sleep(Duration::from_millis(100)).await;
16700        }
16701        if module.exists() {
16702            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16703        }
16704        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16705        std::fs::remove_dir(&root).unwrap();
16706        assert!(
16707            !survived,
16708            "grandchild {grandchild} outlived module teardown"
16709        );
16710        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16711    }
16712}
16713
16714/// A slow live-children record write on the restart path must not stop the
16715/// daemon from answering the replacement module's authentication handshake.
16716#[cfg(all(test, unix))]
16717mod record_write_stall_tests {
16718    use super::*;
16719    use std::sync::atomic::{AtomicBool, Ordering};
16720    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16721
16722    /// `subc-client-rs` gives the whole handshake this long before the module
16723    /// exits with `Timeout { stage: ServerProof }`.
16724    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16725    /// How long the replacement's admission write is held: longer than the
16726    /// module's deadline, standing in for the delay a busy disk can add
16727    /// across the two syncs in that write.
16728    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
16729
16730    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
16731    /// On macOS every module is launched through the privacy trampoline, and
16732    /// a module that is not a `#!` script is admitted only if its running
16733    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
16734    /// the shell soon replaces itself with `/bin/sleep`, which does not match
16735    /// the configured `/bin/sh`. The image is read only after the spawn's
16736    /// live-children record write, and here that write lasted long enough for
16737    /// the shell to have exec'd every time, so each process was refused and
16738    /// killed right after spawn, and the supervisor crash-looped it with
16739    /// backoff until its restart budget ran out. The restart this test
16740    /// measures would then never produce a module at all.
16741    fn sleeping_spec() -> ModuleSpec {
16742        ModuleSpec {
16743            module_id: "record-write-stall".into(),
16744            program: "/bin/sleep".into(),
16745            args: vec!["600".into()],
16746            env: vec![],
16747            reserved: false,
16748            reserved_prefixes: vec![],
16749            protocol: ModuleProtocol::None,
16750            overlap: Default::default(),
16751        }
16752    }
16753
16754    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16755    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
16756        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
16757        let supervisor =
16758            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16759                .with_live_children_record(home.join("live-children.json"));
16760        let module = supervisor.spawn(sleeping_spec()).unwrap();
16761        // `reported_pid` names a process only once it is confirmed as the
16762        // module (on macOS, once the trampoline's image check has passed), so
16763        // a process the daemon is about to refuse and kill never counts as
16764        // the running module, here or in the final wait below.
16765        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
16766            loop {
16767                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
16768                    .unwrap()
16769                    .reported_pid()
16770                {
16771                    return pid;
16772                }
16773                tokio::time::sleep(Duration::from_millis(10)).await;
16774            }
16775        })
16776        .await
16777        .expect("the first process was admitted as the module");
16778
16779        // The daemon's listener, served by this runtime's workers like the
16780        // supervisor's own tasks.
16781        let key = vec![0x42; 32];
16782        let daemon_id = [0x24; 16];
16783        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
16784        let port = listener.local_addr().unwrap().port();
16785        let server = tokio::spawn(crate::server::serve_listener(
16786            listener,
16787            Arc::new(crate::router::Router::with_default_self_handler()),
16788            crate::server::ServerAuth::with_limits(
16789                key.clone(),
16790                daemon_id,
16791                "test",
16792                crate::server::DEFAULT_AUTH_DEADLINE,
16793                4,
16794            ),
16795        ));
16796
16797        // Hold the write that admits the replacement, once, and say when it
16798        // has started.
16799        let (held_tx, held_rx) = std::sync::mpsc::channel();
16800        let armed = AtomicBool::new(true);
16801        supervisor
16802            .child_roster
16803            .set_record_write_hook_for_test(move |live| {
16804                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
16805                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
16806                    let _ = held_tx.send(());
16807                    std::thread::sleep(HELD_RECORD_WRITE);
16808                }
16809            });
16810
16811        // The replacement module, as a separate process would be: its own
16812        // thread and runtime, so only the daemon's runtime can be stalled.
16813        let module_process = std::thread::spawn(move || {
16814            held_rx
16815                .recv_timeout(Duration::from_secs(30))
16816                .expect("the restart reached the replacement's record write");
16817            let conn = ConnectionInfo {
16818                schema: SCHEMA_VERSION,
16819                wire_version: None,
16820                endpoints: vec![Endpoint {
16821                    host: "127.0.0.1".to_owned(),
16822                    port,
16823                }],
16824                key,
16825                daemon_id,
16826                pid: std::process::id(),
16827                daemon_ver: "test".to_owned(),
16828            };
16829            tokio::runtime::Builder::new_current_thread()
16830                .enable_all()
16831                .build()
16832                .unwrap()
16833                .block_on(async {
16834                    let started = Instant::now();
16835                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
16836                        .await
16837                        .unwrap();
16838                    let result =
16839                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
16840                    (result, started.elapsed())
16841                })
16842        });
16843
16844        module.restart(Some(0)).await.unwrap();
16845        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
16846            .await
16847            .unwrap();
16848        assert!(
16849            auth.is_ok(),
16850            "the daemon must answer a module's handshake while a restart waits on the \
16851             live-children record write; got {auth:?} after {elapsed:?}"
16852        );
16853
16854        tokio::time::timeout(Duration::from_secs(20), async {
16855            while lock_snapshot(&module.inner.snapshot)
16856                .unwrap()
16857                .reported_pid()
16858                .is_none_or(|pid| pid == old_pid)
16859            {
16860                tokio::time::sleep(Duration::from_millis(10)).await;
16861            }
16862        })
16863        .await
16864        .expect("the restart published its replacement");
16865        module.stop().await.unwrap();
16866        server.abort();
16867    }
16868}