Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68/// Command-line flag used by supervised modules to find subc.
69///
70/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
71/// The path points at the TCP+key connection file; it is not an ambient signal and
72/// is never inherited by standalone children.
73pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
79/// to contain a real crash loop (which respawns in seconds) and short enough
80/// that unrelated crashes hours apart never accumulate into a permanent stop.
81const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82/// How long a drain waits for already-dispatched requests to finalize before
83/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
84/// not RPC-scale: the original 2s value silently cut nearly every real tool
85/// call at the fence, making the wait-for-finalize design decorative for the
86/// workloads it existed for. Quiescence short-circuits, so an idle module
87/// restarts immediately regardless of this value; the budget is spent only
88/// when a genuine in-flight request is worth finishing. Per-module override:
89/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
90/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
91/// where a stuck request will never settle).
92pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96/// How long a drain waits after one busy-gauge probe completes before it asks
97/// again.
98///
99/// A drain re-probes on this short interval of its own rather than on the
100/// module's health cadence. The default cadence (30 s) equals the default
101/// drain ceiling, so a cadence-paced drain that found the module busy at its
102/// first probe never asked a second time: it waited out the whole ceiling and
103/// forced GOODBYE even after the gauges had read zero. The interval is counted
104/// from the moment the previous probe finished, and probes run one at a time,
105/// so a slow module is never asked more often than it can answer. Periodic
106/// health checks outside a drain keep using the configured cadence.
107const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108/// How long a restart waits for the exited process's output readers.
109///
110/// The restart does not depend on either output reader finishing. A stderr
111/// reader still running at this bound is left running, and its later output goes
112/// into the exited process's own section of the stderr ring (see
113/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
114/// bound no longer decides whether a crash's last lines are kept: under load
115/// the reader may simply not have been scheduled yet, and cutting it there
116/// discarded exactly the lines that explained the crash.
117///
118/// What the bound still decides is when the tail starts reporting
119/// `Incomplete`: a pipe open past it usually means a descendant of the exited
120/// process still holds it, and the tail cannot claim to be whole until that
121/// pipe closes. That is also the only case in which the wait costs the restart
122/// anything, because a pipe with no other holder reaches EOF when the process
123/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
124/// `Captured` at EOF with nothing lost.
125///
126/// The stdout reader carries no ring, only the capture file. It also continues
127/// to EOF after this bound: child exit closes the writer but does not mean the
128/// reader has consumed all buffered bytes. Late output from either pipe can
129/// therefore reach the shared capture file after the successor starts.
130const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
132pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
135/// dropped because the subscriber stopped draining its frames. The detail's
136/// `first_undelivered_cursor` names the first event it did not receive; the
137/// client resubscribes from the last cursor it did receive.
138pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143    deadline: tokio::time::Instant,
144    expected: Option<subc_os::FileIdentity>,
145    trampoline: Option<subc_os::FileIdentity>,
146    script: bool,
147    module_id: String,
148    #[cfg(test)]
149    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
155    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
156    let path = std::env::current_exe()
157        .unwrap()
158        .parent()
159        .unwrap()
160        .parent()
161        .unwrap()
162        .join("privacy-trampoline-fixture");
163    // Without the fixture every macOS spawn is refused, and the tests that
164    // spawn fail later as a module in state Failed, which names the wrong
165    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
166    assert!(
167        path.exists(),
168        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169         --bins --features test-support` or `cargo test -p subc-daemon` first",
170        path.display()
171    );
172    path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177    use std::io::Read;
178    let mut probe = std::process::Command::new(path)
179        .args(["__disclaim-exec", "--probe"])
180        .stdin(Stdio::null())
181        .stdout(Stdio::piped())
182        .stderr(Stdio::piped())
183        .spawn()
184        .map_err(|error| {
185            format!(
186                "privacy trampoline probe failed for {}: {error}",
187                path.display()
188            )
189        })?;
190    let deadline = std::time::Instant::now() + Duration::from_secs(5);
191    let status = loop {
192        match probe.try_wait() {
193            Ok(Some(status)) => break status,
194            Ok(None) if std::time::Instant::now() < deadline => {
195                std::thread::sleep(Duration::from_millis(5))
196            }
197            result => {
198                let _ = probe.kill();
199                let _ = probe.wait();
200                return Err(format!(
201                    "privacy trampoline probe failed or timed out for {}: {result:?}",
202                    path.display()
203                ));
204            }
205        }
206    };
207    let mut answer = String::new();
208    if let Some(stdout) = probe.stdout.take() {
209        let _ = stdout.take(256).read_to_string(&mut answer);
210    }
211    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212        return Ok(());
213    }
214    let mut diagnostic = String::new();
215    if let Some(stderr) = probe.stderr.take() {
216        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217    }
218    let cause = diagnostic
219        .trim()
220        .strip_prefix("ck-subc: own privacy identity refused: ")
221        .unwrap_or("binary does not implement the privacy trampoline protocol");
222    Err(format!(
223        "{cause}: probe of {} exited {status}",
224        path.display()
225    ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230    spec: &ModuleSpec,
231    roster: &ChildRoster,
232) -> Result<
233    (
234        Command,
235        Option<PrivacyExec>,
236        subc_os::privacy_identity::ExecAcknowledgement,
237    ),
238    SuperviseError,
239> {
240    let failure = |cause: String| {
241        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242        SuperviseError::Spawn {
243            program: spec.program.clone(),
244            source: io::Error::other(cause),
245            cgroup_path: None,
246        }
247    };
248    let trampoline = roster.privacy_trampoline().map_err(failure)?;
249    // Resolve PATH with the same environment the Command will receive. For
250    // scripts retain the existing orphan-identity rule: the kernel chooses
251    // the interpreter, and its observed image is the one recorded. Do not
252    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
253    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254        let path = spec
255            .env
256            .iter()
257            .find(|(key, _)| key == "PATH")
258            .map(|(_, value)| std::ffi::OsString::from(value))
259            .or_else(|| std::env::var_os("PATH"))
260            .unwrap_or_else(|| "/usr/bin:/bin".into());
261        std::env::split_paths(&path)
262            .map(|dir| dir.join(&spec.program))
263            .find(|path| path.is_file())
264            .unwrap_or_else(|| spec.program.clone())
265    } else {
266        spec.program.clone()
267    };
268    let expected = subc_os::file_identity(&program);
269    let trampoline_image = subc_os::file_identity(&trampoline);
270    let script = {
271        use std::io::Read;
272        let mut prefix = [0u8; 2];
273        std::fs::File::open(&program)
274            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275    };
276    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277        return Err(failure(
278            "privacy identity module executable is missing or is the trampoline itself".to_string(),
279        ));
280    }
281    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282        .map_err(|error| failure(error.to_string()))?;
283    let reader =
284        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285    let mut command = Command::new(&trampoline);
286    command
287        .arg("__disclaim-exec")
288        .arg(ack.fd().to_string())
289        .arg(&program);
290    ack.install(command.as_std_mut());
291    Ok((
292        command,
293        Some(PrivacyExec {
294            reader,
295            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296            expected,
297            trampoline: trampoline_image,
298            script,
299            module_id: spec.module_id.clone(),
300            #[cfg(test)]
301            confirmation_pause: spec
302                .env
303                .iter()
304                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305                .map(|(_, address)| (address.clone(), None)),
306        }),
307        ack,
308    ))
309}
310
311struct SupervisedChild {
312    child: Child,
313    exec: crate::activity::ExecOwner,
314    #[cfg(windows)]
315    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
316    #[cfg(target_os = "macos")]
317    privacy_exec: Option<PrivacyExec>,
318    /// Set once this launch's exec acknowledgement confirms the module image.
319    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
320    /// `subc_os::privacy_identity`), which then replaces itself with the
321    /// module. The supervisor owns and can kill that pid from spawn, but
322    /// status readers report it only after this latch is set, so nothing
323    /// reports the trampoline's image as the module's.
324    #[cfg(target_os = "macos")]
325    report_ready: Arc<OnceLock<()>>,
326    /// Supervisor failure detail retained for the terminal record, including
327    /// image refusals and first-HELLO timeouts after a successful spawn.
328    spawn_failure: Option<String>,
329    /// The protocol this process was launched with. A reload can store a new
330    /// launch spec with a different protocol, but that takes effect only at the
331    /// next spawn, so this process keeps being handled by the protocol it
332    /// actually speaks.
333    protocol: ModuleProtocol,
334    /// This process's cgroup name: a bounded module/slot label followed by a
335    /// spawn suffix unique to this process (when cgroup placement is on). A
336    /// retired process in a slot may still be draining when a later one is
337    /// spawned into that slot, so the suffix keeps the later process out of
338    /// the retired one's cgroup, which is the domain a kill applies to.
339    #[cfg(target_os = "linux")]
340    module_id: String,
341    #[cfg(target_os = "linux")]
342    cgroup_placement: Option<subc_cgroup::Placement>,
343    /// The job that contains this child and every process it spawns (issue #109).
344    ///
345    /// Dropping this handle is what reaps a surviving tree when no supervisor
346    /// code runs — a daemon crash — because the job carries
347    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
348    ///
349    /// That limit is not crash-only, and the difference is worth knowing: a
350    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
351    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
352    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
353    /// module at once. Before this change they survived that, saw EOF on the
354    /// control socket, and ran their own teardown; Unix keeps that path
355    /// deliberately, so a module can seal a WAL or close a capture rather than
356    /// be killed mid-write. So this trades graceful teardown on every Windows
357    /// daemon stop for containment on a crash, which is the right way round
358    /// today: orphaned GPU workers are a reported, recurring problem, and the
359    /// modules that write most heavily do not run on Windows.
360    ///
361    /// The fix is a real Windows stop path — the daemon draining before it
362    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
363    /// reaches only what the drain left behind, which is what it should reach.
364    #[cfg(windows)]
365    job: Option<subc_jobobject::JobObject>,
366    stdout_pump: Option<JoinHandle<()>>,
367    stderr_pump: Option<StderrPump>,
368    stderr_ring: Arc<Mutex<StderrRing>>,
369    /// Conservative monotonic bound sampled before `command.spawn()`, not when
370    /// identity recording finishes or the spawn generation is published.
371    spawned_at: Instant,
372    /// Ordinary spawns must send their first HELLO within this budget. Once
373    /// accepted, a later disconnect must not restart the registration clock.
374    registration_deadline: Option<Instant>,
375    registration_watch: Option<RegistrationWatch>,
376    spawned_at_ms: u64,
377    spawned_from: PathBuf,
378    launch_spec_digest: LaunchSpecDigest,
379    spawned_file_identity: Option<SpawnedFileIdentity>,
380    process_start_time: Option<u64>,
381    process_identity: Option<ProcessIdentity>,
382    pid: u32,
383    /// This process's entry in the daemon's child roster, released when the
384    /// process is reaped or this handle is dropped.
385    roster_guard: Option<crate::child_roster::RosterGuard>,
386}
387
388impl SupervisedChild {
389    fn registered_since_spawn(&self) -> bool {
390        self.registration_watch
391            .as_ref()
392            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
393    }
394
395    fn id(&self) -> Option<u32> {
396        Some(self.pid)
397    }
398
399    fn process_identity(&self) -> Option<ProcessIdentity> {
400        self.process_identity
401    }
402
403    async fn wait(&mut self) -> io::Result<ExitStatus> {
404        #[cfg(target_os = "macos")]
405        self.confirm_privacy_exec().await;
406        // The roster entry is NOT released here. A daemon shutdown waits for the
407        // roster to empty and then exits the process, so releasing at the reap
408        // let it exit before the exit handler wrote this child's terminal record
409        // (the stderr drain and snapshot update sit in between), and the
410        // shutdown's own `daemon_shutdown` record was intermittently lost. The
411        // caller releases it after recording the exit (`release_roster`), and
412        // dropping the handle releases it too.
413        let result = self.child.wait().await;
414        #[cfg(target_os = "linux")]
415        if result.is_ok() {
416            if let Some(placement) = self.cgroup_placement.as_ref() {
417                cleanup_reaped_cgroup(placement, &self.module_id).await;
418                // Keep ownership while awaiting kernel population changes: a
419                // drain timeout may cancel this wait and then escalate/reap.
420                self.cgroup_placement = None;
421            }
422        }
423        result
424    }
425
426    #[cfg(target_os = "macos")]
427    async fn confirm_privacy_exec(&mut self) {
428        let Some(pending) = &mut self.privacy_exec else {
429            return;
430        };
431        let result = tokio::time::timeout_at(pending.deadline, async {
432            let mut record = Vec::new();
433            loop {
434                let mut ready = pending.reader.readable().await?;
435                let read = ready.try_io(|reader| {
436                    use std::io::Read;
437                    let mut reader = reader.get_ref();
438                    let mut buffer = [0u8; 256];
439                    reader.read(&mut buffer).map(|count| (count, buffer))
440                });
441                match read {
442                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
443                    Ok(Ok((count, buffer))) => {
444                        if record.len() + count > 1024 {
445                            return Err(io::Error::other(
446                                "privacy exec refusal record is too long",
447                            ));
448                        }
449                        record.extend_from_slice(&buffer[..count]);
450                    }
451                    Ok(Err(error)) => return Err(error),
452                    Err(_) => continue,
453                }
454            }
455        })
456        .await;
457        #[cfg(test)]
458        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
459            use tokio::io::{AsyncReadExt, AsyncWriteExt};
460            if stream.is_none() {
461                *stream = Some(
462                    tokio::net::TcpStream::connect(address.as_str())
463                        .await
464                        .unwrap(),
465                );
466                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
467                stream
468                    .as_mut()
469                    .unwrap()
470                    .write_all(&self.pid.to_ne_bytes())
471                    .await
472                    .unwrap();
473            }
474            let mut reply = [0];
475            stream
476                .as_mut()
477                .unwrap()
478                .read_exact(&mut reply)
479                .await
480                .unwrap();
481            pending.confirmation_pause = None;
482            if &reply == b"F" {
483                Ok(Err(io::Error::other("injected late exec refusal")))
484            } else {
485                assert_eq!(&reply, b"X");
486                result
487            }
488        } else {
489            result
490        };
491        // Keep the reader in self across await: select cancellation must not
492        // discard the handshake or reset its original five-second deadline.
493        let pending = self.privacy_exec.as_ref().expect("pending exec");
494        let cause = match result {
495            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
496            Ok(Err(error)) => Some(format!(
497                "privacy identity exec acknowledgement failed: {error}"
498            )),
499            Ok(Ok(record)) if !record.is_empty() => Some(
500                std::str::from_utf8(&record)
501                    .ok()
502                    .and_then(|record| {
503                        record
504                            .trim()
505                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
506                    })
507                    .filter(|cause| !cause.is_empty())
508                    .unwrap_or("invalid privacy exec refusal record")
509                    .to_string(),
510            ),
511            Ok(Ok(_)) => match self.child.try_wait() {
512                // Empty EOF is the exec acknowledgement. A real module may exit
513                // immediately, including with a reserved trampoline status; no
514                // image is admitted, and its ordinary exit contract stays intact.
515                Ok(Some(_status)) => None,
516                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
517                Ok(None) => {
518                    let image = observe_spawned_image(self.pid);
519                    if let Some(image) = image.filter(|image| {
520                        image.executable.is_some()
521                            && image.executable != pending.trampoline
522                            && (image.executable == pending.expected || pending.script)
523                    }) {
524                        if let Err(error) = self
525                            .roster_guard
526                            .as_ref()
527                            .map(|guard| guard.confirm_executable(image))
528                            .transpose()
529                        {
530                            Some(format!(
531                                "could not durably record confirmed module image: {error}"
532                            ))
533                        } else {
534                            let _ = self.report_ready.set(());
535                            info!(module_id = %pending.module_id, pid = self.pid,
536                                "module spawned with own privacy identity (responsibility disclaimed)");
537                            None
538                        }
539                    } else if image.is_none()
540                        || image.is_some_and(|image| image.executable.is_none())
541                    {
542                        // A process can exit between try_wait and the kernel
543                        // image read. Empty EOF already acknowledged exec, so
544                        // preserve that module's ordinary exit rather than
545                        // mislabel a disappearing image as trampoline refusal.
546                        // Keep pending in self across await so cancellation does
547                        // not discard validation or reset its original deadline.
548                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
549                            Ok(Ok(_status)) => None,
550                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
551                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
552                        }
553                    } else {
554                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
555                    }
556                }
557            },
558        };
559        let pending = self.privacy_exec.take().expect("pending exec");
560        // Exec confirmation describes launch success, not roster retirement.
561        // Keep even a reaped child recorded until its terminal history exists.
562        self.exec.0.finish(if cause.is_some() {
563            crate::activity::ExecState::Refused
564        } else {
565            crate::activity::ExecState::CommittedExec
566        });
567        if let Some(cause) = cause {
568            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
569            self.spawn_failure = Some(cause);
570            // No image is admitted on failure. Reach the entire fresh process
571            // group, including a module which spawned a helper before refusal.
572            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
573                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
574            }
575            let _ = self.child.start_kill();
576        }
577    }
578
579    /// Releases this child's daemon-shutdown roster entry once its exit has
580    /// been recorded. The pid is already reaped and free for reuse, so the
581    /// entry must not outlive the record any longer than that.
582    fn release_roster(&mut self) {
583        self.roster_guard = None;
584    }
585
586    /// Kill the child and, where containment is available, its process tree.
587    ///
588    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
589    /// helper process leaked the helper — the Synapse embedding module's CUDA
590    /// worker holds the GPU allocation, so the leak cost VRAM until the next
591    /// restart of something else. Terminating the job reaches grandchildren that
592    /// a tree walk cannot, including one whose parent has already exited and
593    /// been reparented away.
594    ///
595    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
596    /// direct-child kill still decides the outcome, so containment can never
597    /// change whether a module is reported as stopped.
598    fn start_kill(&mut self) -> io::Result<()> {
599        #[cfg(windows)]
600        if let Some(job) = &self.job {
601            if let Err(error) = job.terminate() {
602                debug!(
603                    error = %error,
604                    "job termination failed; the direct-child kill still owns the outcome"
605                );
606            }
607        }
608        #[cfg(target_os = "linux")]
609        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
610        self.child.start_kill()
611    }
612
613    async fn drain_stderr(&mut self, module_id: &str) {
614        if let Some(pump) = self.stdout_pump.take() {
615            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
616        }
617
618        let Some(pump) = self.stderr_pump.take() else {
619            return;
620        };
621        settle_stderr_pump(
622            module_id,
623            &self.stderr_ring,
624            pump,
625            STDERR_PUMP_DRAIN_TIMEOUT,
626        )
627        .await;
628    }
629}
630
631/// Bound the restart's wait, not the lifetime of the exited child's stdout
632/// reader. Slow pipe reads or capture writes must not turn the bound into lost
633/// output; dropping the handle lets the reader finish at EOF, like stderr.
634async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
635    match timeout(bound, &mut pump).await {
636        Ok(Ok(())) => {}
637        Ok(Err(error)) => {
638            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
639        }
640        Err(_) => {
641            drop(pump);
642            warn!(
643                module_id,
644                waited = ?bound,
645                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
646            );
647        }
648    }
649}
650
651/// The reader task for one process's stderr, with the ring generation its
652/// lines are attributed to.
653struct StderrPump {
654    task: JoinHandle<()>,
655    generation: u64,
656}
657
658/// Retire an exited process's stderr reader and wait up to `bound` for it to
659/// reach EOF. A reader still running at the bound is detached, not stopped: it
660/// keeps filling the exited process's section of the ring until its pipe
661/// closes, and the tail reads `Incomplete` until then. See
662/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
663async fn settle_stderr_pump(
664    module_id: &str,
665    ring: &Arc<Mutex<StderrRing>>,
666    pump: StderrPump,
667    bound: Duration,
668) {
669    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
670    let StderrPump {
671        mut task,
672        generation,
673    } = pump;
674    lock().retire_pump(generation);
675    match timeout(bound, &mut task).await {
676        Ok(Ok(())) => {}
677        Ok(Err(err)) => {
678            let mut ring = lock();
679            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
680            ring.finish_pump(generation);
681            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
682        }
683        Err(_) => {
684            // Dropping the handle detaches the task; it ends at EOF on its pipe.
685            drop(task);
686            lock().mark_pump_late(
687                generation,
688                format!(
689                    "stderr of the exited process had not reached EOF {bound:?} after it was \
690                     retired (a descendant may still hold the pipe open); lines it still \
691                     writes are kept in that process's section"
692                ),
693            );
694            warn!(
695                module_id,
696                waited = ?bound,
697                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
698            );
699        }
700    }
701}
702
703fn registration_release_events() -> &'static watch::Sender<u64> {
704    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
705    EVENTS.get_or_init(|| {
706        let (sender, _receiver) = watch::channel(0);
707        sender
708    })
709}
710
711pub(crate) fn notify_registration_release() {
712    let events = registration_release_events();
713    let next_generation = (*events.borrow()).wrapping_add(1);
714    events.send_replace(next_generation);
715}
716
717/// How to launch one singleton module process.
718#[derive(Debug, Clone, PartialEq, Eq)]
719pub struct ModuleSpec {
720    pub module_id: String,
721    pub program: PathBuf,
722    pub args: Vec<String>,
723    pub env: Vec<(String, String)>,
724    /// When true this is a reserved module: each spawn gets a fresh one-time launch
725    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
726    /// process can register this module_id (a security-boundary module like the
727    /// credential vault must not be impersonable while it is down/restarting).
728    pub reserved: bool,
729    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
730    /// Prefixes come from daemon config and must end in `:` before they reach the
731    /// supervisor; the owner module's current spawn nonce authorizes claims under
732    /// each prefix.
733    pub reserved_prefixes: Vec<String>,
734    /// The wire protocol this module speaks, as DECLARED in daemon config.
735    ///
736    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
737    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
738    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
739    /// and NO launch nonce, and a clean exit the daemon did not request is
740    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
741    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
742    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
743    /// because a process ignores an environment variable it does not read.
744    ///
745    /// The argument is the part that cannot be "harmless to a process that
746    /// ignores it": a stock binary exits on an unknown flag before it listens
747    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
748    /// first conformance run against this mode found it. The nonce is withheld
749    /// because a process that will never present it gains nothing from holding
750    /// it, and a secret in the environment of a process that does not need it is
751    /// a leak surface for no benefit.
752    pub protocol: ModuleProtocol,
753    /// Whether two processes of this module may run at once, which is what a
754    /// blue/green swap does for the length of its overlap. Declared in daemon
755    /// config because the daemon must be able to answer it while the module is
756    /// down, and so a module cannot talk itself into it after registering.
757    pub overlap: ModuleOverlap,
758}
759
760/// Only the configured launch inputs enter this value; spawned connection paths,
761/// nonces and inherited environment never do. Its bytes remain daemon-private.
762#[derive(Clone, PartialEq, Eq)]
763pub(crate) struct LaunchSpecDigest([u8; 32]);
764
765impl fmt::Debug for LaunchSpecDigest {
766    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
767        f.write_str("LaunchSpecDigest([redacted])")
768    }
769}
770
771impl LaunchSpecDigest {
772    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
773        fn field(hash: &mut Sha256, bytes: &[u8]) {
774            hash.update((bytes.len() as u64).to_le_bytes());
775            hash.update(bytes);
776        }
777        let mut hash = Sha256::new();
778        if let Some(program) = spec.program.to_str() {
779            field(&mut hash, b"utf8-program");
780            field(&mut hash, program.as_bytes());
781        } else {
782            // The daemon config is UTF-8, but in-process callers may provide
783            // native paths: lossy conversion would collapse distinct programs.
784            #[cfg(unix)]
785            {
786                use std::os::unix::ffi::OsStrExt;
787                field(&mut hash, b"unix-program");
788                field(&mut hash, spec.program.as_os_str().as_bytes());
789            }
790            #[cfg(windows)]
791            {
792                use std::os::windows::ffi::OsStrExt;
793                field(&mut hash, b"windows-program");
794                let bytes = spec
795                    .program
796                    .as_os_str()
797                    .encode_wide()
798                    .flat_map(u16::to_le_bytes)
799                    .collect::<Vec<_>>();
800                field(&mut hash, &bytes);
801            }
802            #[cfg(not(any(unix, windows)))]
803            field(&mut hash, spec.program.to_string_lossy().as_bytes());
804        }
805        hash.update((spec.args.len() as u64).to_le_bytes());
806        for arg in &spec.args {
807            field(&mut hash, arg.as_bytes());
808        }
809        let mut env = spec.env.iter().collect::<Vec<_>>();
810        env.sort_unstable();
811        hash.update((env.len() as u64).to_le_bytes());
812        for (key, value) in env {
813            field(&mut hash, key.as_bytes());
814            field(&mut hash, value.as_bytes());
815        }
816        field(
817            &mut hash,
818            match spec.protocol {
819                ModuleProtocol::Subc => b"subc",
820                ModuleProtocol::None => b"none",
821            },
822        );
823        Self(hash.finalize().into())
824    }
825}
826
827/// Whether a module tolerates a second process of itself running alongside.
828///
829/// Most modules are single-writer on their store (a WAL, a capture log, a
830/// resident index behind a writer barrier), and two processes on one store
831/// corrupt it. So a swap, which overlaps the old and new process by design,
832/// is refused unless the module's config opts in with `overlap: "safe"`.
833#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
834pub enum ModuleOverlap {
835    /// Never run two processes of this module at once. The default.
836    #[default]
837    Exclusive,
838    /// The module has said a second process of itself is harmless for the
839    /// length of a swap.
840    ///
841    /// Declare it only if a second instance can run for a few seconds without
842    /// touching ANY single-writer store: every database, WAL, index, projector
843    /// and scheduled job the module owns. A lease on part of that state is not
844    /// enough. broca's session lease guards WAL appends while its run index, its
845    /// store projector and its archive fold timer (which unlinks live WAL files)
846    /// stay single-writer, so broca is exclusive despite holding a lease. The
847    /// refusal only fires after this has been decided, so the decision is the
848    /// check.
849    Safe,
850}
851
852impl ModuleOverlap {
853    pub fn as_str(self) -> &'static str {
854        match self {
855            Self::Exclusive => "exclusive",
856            Self::Safe => "safe",
857        }
858    }
859}
860
861/// Environment variable telling a spawned module which case it was started
862/// for, before it sends HELLO. Only a swap candidate carries it, as
863/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
864///
865/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
866/// longer because nobody waits on it, while a plain restart must flip ready
867/// quickly because callers see `module_warming` until it does. Absence means
868/// plain restart, the safe reading. The daemon trusts nothing about it; the
869/// candidate is proven by its launch nonce at HELLO.
870pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
871/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
872pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
873/// How long a swap waits for its candidate to register and declare itself
874/// ready when the operator does not say. A module warming as a swap candidate
875/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
876/// daemon allows that plus time to start the process and send HELLO.
877pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
878
879/// Bounded restart policy for crash exits.
880///
881/// `max_restarts` is the number of replacement processes allowed after the
882/// initial spawn WITHIN `window`. After that many crash restarts inside one
883/// window the module enters [`ModuleState::Failed`] and the supervisor stops
884/// the crash loop.
885///
886/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
887/// and that only survived because crashes were rare: a module that crashed
888/// three times across a week was disabled forever by crashes that had nothing
889/// to do with each other. That stopped being survivable once modules began
890/// exiting non-zero whenever the daemon's connection to them drops, because
891/// then every daemon-side connection drop spends a unit of the same budget and
892/// one flappy hour permanently stops a healthy module. Restarts older than
893/// `window` release their slot, so a module that crashed twice yesterday has a
894/// full budget today, while a genuine crash loop -- which is fast by
895/// definition -- still reaches the cap and stops.
896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
897pub struct RestartPolicy {
898    pub max_restarts: u32,
899    /// Base delay before a crash replacement. The actual delay escalates with
900    /// the number of recent crash replacements and is capped by `max_backoff`.
901    pub backoff: Duration,
902    /// Maximum delay before a crash replacement.
903    pub max_backoff: Duration,
904    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
905    /// budget effectively infinite (nothing is ever in-window), which is why
906    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
907    pub window: Duration,
908}
909
910impl RestartPolicy {
911    /// A policy with the default crash window. Callers that care about the
912    /// window say so with [`Self::with_window`]; the ones that do not are
913    /// asking for the standard rate limit, not for no limit.
914    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
915        Self {
916            max_restarts,
917            backoff,
918            max_backoff: DEFAULT_MAX_BACKOFF,
919            window: DEFAULT_RESTART_WINDOW,
920        }
921    }
922
923    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
924        self.max_backoff = max_backoff;
925        self
926    }
927
928    pub fn with_window(mut self, window: Duration) -> Self {
929        self.window = window;
930        self
931    }
932
933    /// Calculate the capped exponential delay for the next crash replacement.
934    /// `restart_in_window` is zero for the first replacement after an operator
935    /// action (restart, reload, re-enable) cleared the crash ring, or after all
936    /// older crash replacements have aged out of the window.
937    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
938        if self.backoff.is_zero() || self.max_backoff.is_zero() {
939            return Duration::ZERO;
940        }
941
942        let mut delay = self.backoff;
943        for _ in 0..restart_in_window {
944            if delay >= self.max_backoff {
945                return self.max_backoff;
946            }
947            delay = delay
948                .checked_mul(10)
949                .unwrap_or(self.max_backoff)
950                .min(self.max_backoff);
951        }
952        delay.min(self.max_backoff)
953    }
954
955    /// The one sentence that explains a budget-exhausted stop, used for both the
956    /// log line and the terminal record so the two cannot drift. It names the
957    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
958    /// exactly what this budget is not.
959    fn budget_exhausted_detail(&self) -> String {
960        format!(
961            "crash budget exhausted: max_restarts={} within window_secs={}",
962            self.max_restarts,
963            self.window.as_secs()
964        )
965    }
966}
967
968impl Default for RestartPolicy {
969    fn default() -> Self {
970        Self {
971            max_restarts: DEFAULT_MAX_RESTARTS,
972            backoff: DEFAULT_BACKOFF,
973            max_backoff: DEFAULT_MAX_BACKOFF,
974            window: DEFAULT_RESTART_WINDOW,
975        }
976    }
977}
978
979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
980struct CrashRestartSchedule {
981    restart_in_window: u32,
982    delay: Duration,
983}
984
985/// Whether the daemon itself will bring this module back after the exit being
986/// handled: it is enabled AND its in-window crash restarts are below the cap.
987///
988/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
989/// the window are dropped here rather than by a timer, so the count is right
990/// the moment somebody asks and no bookkeeping runs for idle modules.
991fn daemon_will_restart(
992    state: &mut SupervisorSnapshot,
993    policy: &RestartPolicy,
994    now: Instant,
995) -> bool {
996    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
997}
998
999const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1000const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1001const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1002const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1003
1004#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1005pub enum HealthAction {
1006    Report,
1007    Restart,
1008    Alert,
1009}
1010
1011impl fmt::Display for HealthAction {
1012    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1013        f.write_str(match self {
1014            Self::Report => "report",
1015            Self::Restart => "restart",
1016            Self::Alert => "alert",
1017        })
1018    }
1019}
1020
1021#[derive(Debug, Clone, PartialEq, Eq)]
1022pub struct HealthConfig {
1023    /// Optional loopback HTTP endpoint for a managed non-wire process.
1024    /// Changing it applies live on rescan; the process protocol changes only
1025    /// at its next spawn.
1026    pub http: Option<String>,
1027    pub cadence: Duration,
1028    pub deadline: Duration,
1029    pub failure_threshold: u32,
1030    pub on_degraded: HealthAction,
1031    pub on_failing: HealthAction,
1032    pub critical: bool,
1033}
1034
1035impl Default for HealthConfig {
1036    fn default() -> Self {
1037        Self {
1038            http: None,
1039            cadence: DEFAULT_HEALTH_CADENCE,
1040            deadline: DEFAULT_HEALTH_DEADLINE,
1041            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1042            on_degraded: HealthAction::Report,
1043            on_failing: HealthAction::Report,
1044            critical: false,
1045        }
1046    }
1047}
1048
1049/// The supervisor's view of one module's health, relayed to clients over
1050/// channel-0 and rendered by `ck health`.
1051///
1052/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1053/// stated here rather than only at the wire type a consumer reads. A reader can
1054/// look up what `None` means; only a writer can silently change it, and the
1055/// writer has no reason to go looking at a downstream contract before editing.
1056///
1057/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1058/// back to `None` on re-registration precisely so a respawned module does not
1059/// carry its predecessor's timestamp — so an old value and an absent one call for
1060/// opposite readings, and anything that defaulted this to a number would make a
1061/// never-probed module indistinguishable from one probed at the epoch.
1062///
1063/// `detail` and `metrics` are `None` when the module published none on this
1064/// probe, which does not mean it reported nothing wrong — it is also the shape
1065/// when the probe never reached it. `last_probe_ms` is what separates those.
1066#[derive(Debug, Clone, PartialEq)]
1067pub struct ModuleHealthStatus {
1068    pub status: SupervisorHealthStatus,
1069    pub last_probe_ms: Option<u64>,
1070    pub detail: Option<String>,
1071    pub metrics: Option<Value>,
1072    pub consecutive_failures: u32,
1073    /// Number of replies received after a recurring health probe's deadline.
1074    /// Unlike a timeout, every increment proves the module was alive.
1075    pub late_answer_count: u64,
1076    /// End-to-end latency of the newest late reply, measured from probe start.
1077    pub last_late_answer_latency_ms: Option<u64>,
1078    pub last_action: Option<String>,
1079    /// Set together with `last_action`; the pair moves as one, and both being
1080    /// absent means no escalation has ever been taken rather than that the last
1081    /// one succeeded.
1082    pub last_action_ms: Option<u64>,
1083}
1084
1085impl Default for ModuleHealthStatus {
1086    fn default() -> Self {
1087        Self {
1088            status: SupervisorHealthStatus::Unknown,
1089            last_probe_ms: None,
1090            detail: None,
1091            metrics: None,
1092            consecutive_failures: 0,
1093            late_answer_count: 0,
1094            last_late_answer_latency_ms: None,
1095            last_action: None,
1096            last_action_ms: None,
1097        }
1098    }
1099}
1100
1101/// Typed lifecycle state for a supervised module.
1102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1103pub enum ModuleState {
1104    Starting,
1105    Running,
1106    Unresponsive,
1107    Restarting,
1108    Draining,
1109    Stopped,
1110    Failed,
1111    Disabled,
1112}
1113
1114impl fmt::Display for ModuleState {
1115    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1116        f.write_str(match self {
1117            Self::Starting => "starting",
1118            Self::Running => "running",
1119            Self::Unresponsive => "unresponsive",
1120            Self::Restarting => "restarting",
1121            Self::Draining => "draining",
1122            Self::Stopped => "stopped",
1123            Self::Failed => "failed",
1124            Self::Disabled => "disabled",
1125        })
1126    }
1127}
1128
1129/// Supervisor classification of a child-process exit.
1130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1131pub enum ExitKind {
1132    Clean,
1133    Crash,
1134    DeliberateSeverance,
1135}
1136
1137impl From<ExitKind> for TerminalExitKind {
1138    fn from(kind: ExitKind) -> Self {
1139        match kind {
1140            ExitKind::Clean => Self::Clean,
1141            ExitKind::Crash => Self::Crash,
1142            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1143        }
1144    }
1145}
1146
1147/// Exact process identity retained when a supervised module registers its
1148/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub(crate) struct ProcessIdentity {
1151    pub(crate) pid: u32,
1152    pub(crate) start_time: u64,
1153}
1154
1155/// Last observed child exit, if any.
1156#[derive(Debug, Clone, PartialEq, Eq)]
1157pub struct ExitReport {
1158    pub kind: ExitKind,
1159    pub code: Option<i32>,
1160    pub signal: Option<i32>,
1161    pub at_ms: u64,
1162}
1163
1164/// Point-in-time module status answerable by subc without forwarding to the
1165/// module process.
1166#[derive(Debug, Clone, PartialEq)]
1167pub struct ModuleStatus {
1168    pub module_id: String,
1169    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1170    pub waiting_for_previous_pid: Option<u32>,
1171    /// Why the previous instance still prevents a new start.
1172    pub waiting_for_previous_detail: Option<String>,
1173    /// The daemon's Windows per-spawn record: `pipe` after the process it
1174    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1175    /// List/status reports use this record, not the source the module sends
1176    /// in its registration message's manifest.provenance.launch_nonce_source.
1177    pub launch_nonce_source: Option<String>,
1178    pub state: ModuleState,
1179    pub enabled: bool,
1180    pub process_alive: bool,
1181    pub registration_active: bool,
1182    /// The module's declared wire protocol, carried beside `live` because it is
1183    /// what makes `live` readable: the two fields answer one question together.
1184    /// While a process is alive this is its launch declaration, not a later
1185    /// pending-reload edit. When down it is the configured next launch protocol.
1186    pub protocol: ModuleProtocol,
1187    /// Whether the module is serving, under the strongest definition the daemon
1188    /// can assert for its protocol.
1189    ///
1190    /// A subc module must also be REGISTERED: its process being alive says
1191    /// nothing about whether it can take a request. A `protocol: "none"` module
1192    /// never registers, so that term is dropped and this falls back to "enabled,
1193    /// running, and the process the daemon launched is alive" -- which is all
1194    /// the daemon observes about a process that speaks no subc wire. It stays a
1195    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1196    /// rather than printing it bare.
1197    pub live: bool,
1198    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1199    /// restarts have already released their slot, so this count can go down
1200    /// without anybody touching the module.
1201    pub restart_count: u32,
1202    /// Replacement processes spawned over this module's entire supervisor lifetime;
1203    /// unlike `restart_count`, this value is never reset by an operator action
1204    /// and never falls out of a window.
1205    pub lifetime_restarts: u32,
1206    pub spawn_generation: u64,
1207    /// The budget `restart_count` is spent against. Carried alongside the count
1208    /// because the count alone does not say how close the module is to being
1209    /// disabled, and reporting one without the other is what makes an
1210    /// about-to-be-retired module look ordinary.
1211    pub max_restarts: u32,
1212    /// The span `restart_count` is counted over. Carried with the pair above for
1213    /// the same reason they are carried together: "2 of 3" means one thing for a
1214    /// ten-minute window and something else entirely for a lifetime.
1215    pub restart_window: Duration,
1216    /// Effective drain and restart timing policy used by this running module.
1217    /// These values are carried together with the restart budget so status
1218    /// readers can compare configured intent with what the supervisor applied.
1219    pub drain_timeout: Duration,
1220    pub restart_backoff: Duration,
1221    pub restart_max_backoff: Duration,
1222    /// The module's process. On macOS this stays absent while the `ck-subc`
1223    /// launch trampoline is still running in that pid, and appears once the
1224    /// exec acknowledgement confirms the module image has replaced it. Launch
1225    /// time and the supervisor's own hold on the process are unaffected.
1226    pub pid: Option<u32>,
1227    pub spawned_at_ms: Option<u64>,
1228    pub spawned_from: Option<PathBuf>,
1229    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1230    pub process_start_time: Option<u64>,
1231    pub last_exit: Option<ExitReport>,
1232    pub health: ModuleHealthStatus,
1233}
1234
1235#[cfg(test)]
1236#[derive(Debug, Clone, Copy, PartialEq)]
1237struct ActorSelectCheckpoint {
1238    generation: u64,
1239    turn: u64,
1240    registered_connection: Option<ConnectionId>,
1241    next_probe_at: Option<Instant>,
1242    wake_after: Option<Duration>,
1243}
1244
1245#[derive(Debug, Clone, PartialEq)]
1246struct SupervisorSnapshot {
1247    failed_retry: FailedRetry,
1248    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1249    #[cfg(windows)]
1250    pipe_consumed: Option<Arc<OnceLock<()>>>,
1251    /// Counts running-child loop turns, not executor polls of a parked wait.
1252    #[cfg(test)]
1253    actor_turns: u64,
1254    /// Running select after exec confirmation and with no unconsumed registry
1255    /// event. Recording a spawn or starting a loop turn is not this barrier.
1256    #[cfg(test)]
1257    actor_select: Option<ActorSelectCheckpoint>,
1258    state: ModuleState,
1259    enabled: bool,
1260    process_alive: bool,
1261    spawned_protocol: Option<ModuleProtocol>,
1262    spawn_failure: Option<String>,
1263    waiting_for_previous_pid: Option<u32>,
1264    waiting_for_previous_detail: Option<String>,
1265    /// When each crash restart was spent, oldest first. This IS the crash
1266    /// budget: its in-window length is the count an operator sees and the count
1267    /// the restart decision is made against, so there is no second counter that
1268    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1269    /// operator actions that used to zero the old lifetime counter.
1270    crash_restarts: VecDeque<Instant>,
1271    lifetime_restarts: u32,
1272    /// Successful child spawns in this daemon incarnation.
1273    ///
1274    /// `lifetime_restarts` was considered and rejected: it starts at zero
1275    /// (line 640), successful initial/operator spawns in `set_running` do not
1276    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1277    /// increments before a successful replacement exists (lines 604, 3846,
1278    /// and 3921), so a failed spawn can consume it. This counter moves only
1279    /// when a live PID is accepted below.
1280    spawn_generation: u64,
1281    pid: Option<u32>,
1282    #[cfg(target_os = "macos")]
1283    report_ready: Option<Arc<OnceLock<()>>>,
1284    /// Last reaped child, retained after current process facts are cleared.
1285    reaped_pid: Option<u32>,
1286    /// Whether a replacement has already been scheduled for the current process.
1287    respawn_pending: bool,
1288    /// A second restart is waiting for the replacement already scheduled.
1289    coalesced_restart_pending: bool,
1290    spawned_at_ms: Option<u64>,
1291    spawned_from: Option<PathBuf>,
1292    launch_spec_digest: Option<LaunchSpecDigest>,
1293    spawned_file_identity: Option<SpawnedFileIdentity>,
1294    process_start_time: Option<u64>,
1295    deliberate_severance: Option<ProcessIdentity>,
1296    last_exit: Option<ExitReport>,
1297    /// Diagnostic attached to the next drain's terminal record, if any.
1298    drain_disposition_detail: Option<String>,
1299    health: ModuleHealthStatus,
1300    /// Whether the current process was started as a swap candidate and so
1301    /// lives in the module's alternate cgroup. The next swap's candidate takes
1302    /// the other one, so the two processes of a swap never share a cgroup. A
1303    /// plain spawn always uses the primary cgroup.
1304    in_alternate_slot: bool,
1305    /// Whether the current `Draining` state ends in a replacement process
1306    /// (restart, reload, health restart) rather than a stop. Only meaningful
1307    /// while `state` is `Draining`; every entry into that state rewrites it.
1308    /// It is what lets route.open answer the retryable `module_reloading` to a
1309    /// consumer that reaches a still-registered process mid-restart, instead of
1310    /// the `supervisor_not_live` a stop or disable deserves.
1311    draining_to_replace: bool,
1312    /// Whether a configuration update has been applied since the current
1313    /// process was spawned, so that process runs an older spec than the one
1314    /// the supervisor now holds. A queued restart is only coalesced into a
1315    /// fresher process when this is false: a restart requested to pick up a
1316    /// new configuration must not be satisfied by a process that predates it.
1317    configuration_updated_since_spawn: bool,
1318}
1319
1320impl SupervisorSnapshot {
1321    /// The pid that status, provenance and resource readings may report. While
1322    /// the launch trampoline still runs in the pid, reading its executable or
1323    /// resource use would describe `ck-subc`, not the module, so none is
1324    /// reported until the exec acknowledgement confirms the module image.
1325    fn reported_pid(&self) -> Option<u32> {
1326        #[cfg(target_os = "macos")]
1327        if self
1328            .report_ready
1329            .as_ref()
1330            .is_some_and(|ready| ready.get().is_none())
1331        {
1332            return None;
1333        }
1334        self.pid
1335    }
1336
1337    fn starting() -> Self {
1338        Self::new(ModuleState::Starting, true)
1339    }
1340
1341    fn disabled() -> Self {
1342        Self::new(ModuleState::Disabled, false)
1343    }
1344
1345    /// Crash restarts still inside `window`, having dropped the ones that are
1346    /// not. Pruning on read is what makes the budget a rate: an instant older
1347    /// than the window stops holding a slot the moment anybody counts.
1348    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1349        while let Some(oldest) = self.crash_restarts.front() {
1350            if now.duration_since(*oldest) > window {
1351                self.crash_restarts.pop_front();
1352            } else {
1353                break;
1354            }
1355        }
1356        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1357    }
1358
1359    /// Spend one unit of the crash budget and record the restart in the ledger.
1360    ///
1361    /// The ring is bounded by the cap because more than `max_restarts` in-window
1362    /// instants can never be reached (the caller refuses the restart first), so
1363    /// anything beyond that is an unbounded queue waiting to happen.
1364    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1365        self.crash_restarts.push_back(now);
1366        while self.crash_restarts.len() > policy.max_restarts as usize {
1367            self.crash_restarts.pop_front();
1368        }
1369        self.lifetime_restarts += 1;
1370    }
1371
1372    /// Reserve one crash-restart slot and calculate the delay before respawning.
1373    /// The count is captured before recording this restart, so the first retry
1374    /// uses the base delay and each later in-window retry escalates once.
1375    fn next_crash_restart(
1376        &mut self,
1377        policy: &RestartPolicy,
1378        now: Instant,
1379    ) -> Option<CrashRestartSchedule> {
1380        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1381        if restart_in_window >= policy.max_restarts {
1382            return None;
1383        }
1384        self.record_crash_restart(policy, now);
1385        Some(CrashRestartSchedule {
1386            restart_in_window,
1387            delay: policy.delay_for_restart(restart_in_window),
1388        })
1389    }
1390
1391    /// Give the module its full budget back, as an operator restart, reload, or
1392    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1393    /// ledger of what actually happened, and an operator action does not unmake
1394    /// the crashes.
1395    fn clear_crash_restarts(&mut self) {
1396        self.crash_restarts.clear();
1397    }
1398
1399    fn new(state: ModuleState, enabled: bool) -> Self {
1400        Self {
1401            failed_retry: FailedRetry::default(),
1402            exec_outcome: None,
1403            #[cfg(test)]
1404            actor_turns: 0,
1405            #[cfg(test)]
1406            actor_select: None,
1407            state,
1408            enabled,
1409            process_alive: false,
1410            #[cfg(windows)]
1411            pipe_consumed: None,
1412            spawned_protocol: None,
1413            spawn_failure: None,
1414            waiting_for_previous_pid: None,
1415            waiting_for_previous_detail: None,
1416            crash_restarts: VecDeque::new(),
1417            lifetime_restarts: 0,
1418            spawn_generation: 0,
1419            pid: None,
1420            #[cfg(target_os = "macos")]
1421            report_ready: None,
1422            reaped_pid: None,
1423            respawn_pending: false,
1424            coalesced_restart_pending: false,
1425            spawned_at_ms: None,
1426            spawned_from: None,
1427            launch_spec_digest: None,
1428            spawned_file_identity: None,
1429            process_start_time: None,
1430            deliberate_severance: None,
1431            last_exit: None,
1432            drain_disposition_detail: None,
1433            health: ModuleHealthStatus::default(),
1434            in_alternate_slot: false,
1435            draining_to_replace: false,
1436            configuration_updated_since_spawn: false,
1437        }
1438    }
1439}
1440
1441type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1442
1443type SpawnSubscriberKey = (ConnectionId, u64);
1444
1445#[derive(Debug)]
1446struct SpawnSubscriber {
1447    version: u8,
1448    frames: mpsc::Sender<Frame>,
1449    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1450    /// from which event. The full frame channel cannot carry that news, so it
1451    /// travels beside it; see `SpawnEventFeed::subscribe`.
1452    lagged: Option<oneshot::Sender<SpawnCursor>>,
1453}
1454
1455#[derive(Debug)]
1456struct SpawnEventState {
1457    daemon_incarnation: String,
1458    seq: u64,
1459    capacity: usize,
1460    live: HashMap<String, LiveSpawn>,
1461    generations: HashMap<String, u64>,
1462    events: VecDeque<SpawnEvent>,
1463    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1464}
1465
1466impl Default for SpawnEventState {
1467    fn default() -> Self {
1468        Self {
1469            daemon_incarnation: "unconfigured".to_string(),
1470            seq: 0,
1471            capacity: SPAWN_EVENT_RING_CAPACITY,
1472            live: HashMap::new(),
1473            generations: HashMap::new(),
1474            events: VecDeque::new(),
1475            subscribers: HashMap::new(),
1476        }
1477    }
1478}
1479
1480#[derive(Debug, Clone, Default)]
1481struct SpawnEventFeed(
1482    Arc<Mutex<SpawnEventState>>,
1483    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1484);
1485
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub(crate) enum SpawnSubscribeRefusal {
1488    ForeignIncarnation { current: String },
1489    TooOld { oldest: SpawnCursor },
1490    Frame(String),
1491}
1492
1493impl SpawnEventFeed {
1494    fn configure_incarnation(&self, daemon_incarnation: String) {
1495        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1496        state.daemon_incarnation = daemon_incarnation;
1497        state.seq = 0;
1498        state.live.clear();
1499        state.generations.clear();
1500        state.events.clear();
1501        state.subscribers.clear();
1502    }
1503
1504    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1505        SpawnCursor {
1506            daemon_incarnation: state.daemon_incarnation.clone(),
1507            seq: state.seq,
1508        }
1509    }
1510
1511    fn snapshot(&self) -> SpawnSnapshot {
1512        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1513        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1514        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1515        SpawnSnapshot {
1516            cursor: Self::cursor(&state),
1517            ring_bound: state.capacity as u64,
1518            live,
1519        }
1520    }
1521
1522    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1523        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1524        let generation = state
1525            .generations
1526            .get(module_id)
1527            .copied()
1528            .unwrap_or(0)
1529            .checked_add(1)
1530            .expect("spawn generation exhausted");
1531        state.generations.insert(module_id.to_string(), generation);
1532        let live = LiveSpawn {
1533            module_id: module_id.to_string(),
1534            spawn_generation: generation,
1535            pid,
1536            spawned_at_ms,
1537        };
1538        state.live.insert(module_id.to_string(), live);
1539        Self::emit_locked(
1540            &mut state,
1541            SpawnEventKind::Spawned,
1542            module_id.to_string(),
1543            generation,
1544            pid,
1545            None,
1546            None,
1547        );
1548        generation
1549    }
1550
1551    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1552        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1553        let Some(live) = state.live.remove(module_id) else {
1554            warn!(
1555                module_id,
1556                "terminal record had no live spawn event identity"
1557            );
1558            return;
1559        };
1560        Self::emit_locked(
1561            &mut state,
1562            SpawnEventKind::Exited,
1563            module_id.to_string(),
1564            live.spawn_generation,
1565            live.pid,
1566            exit_code,
1567            exit_signal,
1568        );
1569    }
1570
1571    /// Report the exit of a process that a swap has already replaced.
1572    ///
1573    /// `emit_exited` removes the module's live entry, which after a swap's
1574    /// cutover describes the promoted candidate, not the old process now
1575    /// exiting. This emits the old generation's exit and leaves the live entry
1576    /// alone unless it still names that generation.
1577    fn emit_superseded_exited(
1578        &self,
1579        module_id: &str,
1580        spawn_generation: u64,
1581        pid: u32,
1582        exit_code: Option<i32>,
1583        exit_signal: Option<i32>,
1584    ) {
1585        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1586        if state
1587            .live
1588            .get(module_id)
1589            .is_some_and(|live| live.spawn_generation == spawn_generation)
1590        {
1591            state.live.remove(module_id);
1592        }
1593        Self::emit_locked(
1594            &mut state,
1595            SpawnEventKind::Exited,
1596            module_id.to_string(),
1597            spawn_generation,
1598            pid,
1599            exit_code,
1600            exit_signal,
1601        );
1602    }
1603
1604    #[allow(clippy::too_many_arguments)]
1605    fn emit_locked(
1606        state: &mut SpawnEventState,
1607        kind: SpawnEventKind,
1608        module_id: String,
1609        spawn_generation: u64,
1610        pid: u32,
1611        exit_code: Option<i32>,
1612        exit_signal: Option<i32>,
1613    ) {
1614        state.seq = state
1615            .seq
1616            .checked_add(1)
1617            .expect("spawn event sequence exhausted");
1618        let event = SpawnEvent {
1619            cursor: Self::cursor(state),
1620            kind,
1621            module_id,
1622            spawn_generation,
1623            pid,
1624            exit_code,
1625            exit_signal,
1626        };
1627        state.events.push_back(event.clone());
1628        while state.events.len() > state.capacity {
1629            state.events.pop_front();
1630        }
1631        let body = match serde_json::to_vec(&event) {
1632            Ok(body) => body,
1633            Err(error) => {
1634                error!(%error, "failed to serialize supervisor spawn event");
1635                return;
1636            }
1637        };
1638        state.subscribers.retain(|(connection_id, corr), subscriber| {
1639            let frame = Frame::build_with_version(
1640                subscriber.version,
1641                FrameType::StreamData,
1642                control_flags(),
1643                0,
1644                0,
1645                *corr,
1646                body.clone(),
1647            );
1648            match frame {
1649                Ok(frame) => {
1650                    if subscriber.frames.try_send(frame).is_ok() {
1651                        true
1652                    } else {
1653                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1654                        if let Some(lagged) = subscriber.lagged.take() {
1655                            let _ = lagged.send(event.cursor.clone());
1656                        }
1657                        false
1658                    }
1659                }
1660                Err(error) => {
1661                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1662                    false
1663                }
1664            }
1665        });
1666    }
1667
1668    fn subscribe(
1669        &self,
1670        connection_id: ConnectionId,
1671        corr: u64,
1672        version: u8,
1673        since: Option<SpawnCursor>,
1674        sink: FrameSink,
1675    ) -> Result<(), SpawnSubscribeRefusal> {
1676        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1677        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1678        {
1679            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1680            let replay = if let Some(since) = since {
1681                if since.daemon_incarnation != state.daemon_incarnation {
1682                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1683                        current: state.daemon_incarnation.clone(),
1684                    });
1685                }
1686                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1687                    if since.seq < oldest.seq.saturating_sub(1) {
1688                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1689                    }
1690                }
1691                state
1692                    .events
1693                    .iter()
1694                    .filter(|event| event.cursor.seq > since.seq)
1695                    .cloned()
1696                    .collect::<Vec<_>>()
1697            } else {
1698                Vec::new()
1699            };
1700            for event in replay {
1701                let body = serde_json::to_vec(&event)
1702                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1703                let frame = Frame::build_with_version(
1704                    version,
1705                    FrameType::StreamData,
1706                    control_flags(),
1707                    0,
1708                    0,
1709                    corr,
1710                    body,
1711                )
1712                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1713                frames
1714                    .try_send(frame)
1715                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716            }
1717            state.subscribers.insert(
1718                (connection_id, corr),
1719                SpawnSubscriber {
1720                    version,
1721                    frames,
1722                    lagged: Some(lagged),
1723                },
1724            );
1725        }
1726        // The lagged terminal is sent here, by the forwarder, rather than by
1727        // the emitter: at the moment of the drop the subscriber's own channel
1728        // is full, and writing to the connection sink directly from the emitter
1729        // would put the Error AHEAD of the events still queued in that channel
1730        // (and the emitter holds the feed lock, so it cannot await the sink).
1731        // Dropping the subscriber drops the only sender, so `recv` drains every
1732        // queued event and then returns `None`; only then is the Error sent, so
1733        // the client sees each event it can keep, then the reason it was cut.
1734        // Cancel and connection removal drop the oneshot unsent, so they end
1735        // the stream with no Error.
1736        tokio::spawn(async move {
1737            while let Some(frame) = receiver.recv().await {
1738                if sink.send(frame).await.is_err() {
1739                    return;
1740                }
1741            }
1742            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1743                return;
1744            };
1745            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1746                Ok(frame) => {
1747                    let _ = sink.send(frame).await;
1748                }
1749                Err(error) => {
1750                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1751                }
1752            }
1753        });
1754        Ok(())
1755    }
1756
1757    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1758        let Some(subscriber) = self
1759            .0
1760            .lock()
1761            .unwrap_or_else(|p| p.into_inner())
1762            .subscribers
1763            .remove(&(connection_id, corr))
1764        else {
1765            return false;
1766        };
1767        if let Ok(frame) = Frame::build_with_version(
1768            subscriber.version,
1769            FrameType::StreamEnd,
1770            control_flags(),
1771            0,
1772            0,
1773            corr,
1774            Vec::new(),
1775        ) {
1776            tokio::spawn(async move {
1777                let _ = subscriber.frames.send(frame).await;
1778            });
1779        }
1780        true
1781    }
1782
1783    fn remove_connection(&self, connection_id: ConnectionId) {
1784        self.0
1785            .lock()
1786            .unwrap_or_else(|p| p.into_inner())
1787            .subscribers
1788            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1789    }
1790
1791    #[cfg(any(test, feature = "test-support"))]
1792    fn set_capacity(&self, capacity: usize) {
1793        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1794    }
1795
1796    #[cfg(any(test, feature = "test-support"))]
1797    fn subscriber_count(&self) -> usize {
1798        self.0
1799            .lock()
1800            .unwrap_or_else(|p| p.into_inner())
1801            .subscribers
1802            .len()
1803    }
1804}
1805
1806/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1807/// The terminal Error a lagged spawn subscriber receives after its queued events.
1808fn spawn_subscriber_lagged_frame(
1809    version: u8,
1810    corr: u64,
1811    first_undelivered: SpawnCursor,
1812) -> Result<Frame, String> {
1813    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1814    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1815        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1816        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1817            .to_string(),
1818        detail: Some(serde_json::json!({
1819            "first_undelivered_cursor": first_undelivered
1820        })),
1821    })
1822    .map_err(|error| error.to_string())?;
1823    Frame::build_with_version(
1824        version,
1825        FrameType::Error,
1826        crate::router::daemon_error_flags(),
1827        0,
1828        0,
1829        corr,
1830        body,
1831    )
1832    .map_err(|error| error.to_string())
1833}
1834
1835pub trait ModuleProcessLiveness: Send + Sync {
1836    fn process_live(&self, module_id: &str) -> Option<bool>;
1837
1838    /// Whether the supervisor is replacing this module's process right now: an
1839    /// operator restart or reload, a health restart, or a crash respawn whose
1840    /// backoff is running. A module in that state is not live, but a consumer
1841    /// refused now should retry shortly rather than treat the target as gone.
1842    /// Stopped, failed, and disabled modules are not replacing.
1843    fn process_replacing(&self, _module_id: &str) -> bool {
1844        false
1845    }
1846}
1847
1848/// Shared process-liveness registry keyed by supervised `module_id`.
1849#[derive(Debug, Clone, Default)]
1850pub struct SupervisorProcessLiveness {
1851    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1852}
1853
1854impl SupervisorProcessLiveness {
1855    pub fn new() -> Self {
1856        Self::default()
1857    }
1858
1859    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1860        let mut snapshots = self
1861            .snapshots
1862            .lock()
1863            .unwrap_or_else(|poisoned| poisoned.into_inner());
1864        snapshots.insert(module_id, snapshot);
1865    }
1866
1867    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1868        let mut snapshots = self
1869            .snapshots
1870            .lock()
1871            .unwrap_or_else(|poisoned| poisoned.into_inner());
1872        let is_current = snapshots
1873            .get(module_id)
1874            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1875            .unwrap_or(false);
1876        if is_current {
1877            snapshots.remove(module_id);
1878        }
1879    }
1880}
1881
1882impl ModuleProcessLiveness for SupervisorProcessLiveness {
1883    fn process_live(&self, module_id: &str) -> Option<bool> {
1884        let snapshot = {
1885            let snapshots = self
1886                .snapshots
1887                .lock()
1888                .unwrap_or_else(|poisoned| poisoned.into_inner());
1889            snapshots.get(module_id).cloned()
1890        }?;
1891        let snapshot = snapshot
1892            .lock()
1893            .unwrap_or_else(|poisoned| poisoned.into_inner());
1894        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1895    }
1896
1897    fn process_replacing(&self, module_id: &str) -> bool {
1898        let Some(snapshot) = self
1899            .snapshots
1900            .lock()
1901            .unwrap_or_else(|poisoned| poisoned.into_inner())
1902            .get(module_id)
1903            .cloned()
1904        else {
1905            return false;
1906        };
1907        let snapshot = snapshot
1908            .lock()
1909            .unwrap_or_else(|poisoned| poisoned.into_inner());
1910        snapshot.enabled
1911            && match snapshot.state {
1912                ModuleState::Restarting => true,
1913                ModuleState::Draining => snapshot.draining_to_replace,
1914                ModuleState::Starting
1915                | ModuleState::Running
1916                | ModuleState::Unresponsive
1917                | ModuleState::Stopped
1918                | ModuleState::Failed
1919                | ModuleState::Disabled => false,
1920            }
1921    }
1922}
1923
1924#[cfg(test)]
1925#[derive(Debug, Default)]
1926struct ReloadExitRecordGate {
1927    reached: tokio::sync::Notify,
1928    resume: tokio::sync::Notify,
1929}
1930
1931#[derive(Debug, Clone, Copy)]
1932enum RespawnKind {
1933    Spawn,
1934    Reload,
1935}
1936
1937#[derive(Debug, Clone, Copy)]
1938struct PendingRespawn {
1939    deadline: Instant,
1940    kind: RespawnKind,
1941    daemon_initiated: bool,
1942}
1943
1944type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1945
1946#[derive(Debug, Clone)]
1947struct SupervisorRuntimeConfig {
1948    registry: Arc<Registry>,
1949    /// Retained across an operation's deferred backoff, until its child starts.
1950    recovery_is_daemon: bool,
1951    start_requester: Option<Principal>,
1952    pending_activity_restart: bool,
1953    provenance_probe: ExecutableIdentityProbe,
1954    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1955    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1956    /// A reload acknowledges completion only after its replacement registers.
1957    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1958    restart_policy: RestartPolicy,
1959    register_timeout: Duration,
1960    /// This module's RESOLVED drain budget: per-module config when present,
1961    /// else `default_drain_timeout`.
1962    drain_timeout: Duration,
1963    /// Shared with the status handle so the attested value changes atomically
1964    /// when a rescan updates the running drain policy.
1965    effective_drain_timeout: Arc<Mutex<Duration>>,
1966    /// The supervisor-wide fallback, kept so a configuration update that
1967    /// REMOVES the per-module override can re-resolve to it.
1968    default_drain_timeout: Duration,
1969    health: HealthConfig,
1970    connection_file_path: Option<PathBuf>,
1971    capture_logs_dir: Option<PathBuf>,
1972    forwarding: Option<Arc<ForwardingTable>>,
1973    /// The shared handle, so every spawn path (initial, restart, reload) records the
1974    /// reserved-module launch nonce the HELLO verifier checks against.
1975    supervisor_handle: Option<SupervisorHandle>,
1976    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1977    /// status queries.
1978    ///
1979    /// One ring per module, held across every respawn. The lines explaining an exit
1980    /// are written BEFORE that exit, so a ring recreated per process would be empty
1981    /// exactly when it is asked for.
1982    stderr_ring: Arc<Mutex<StderrRing>>,
1983    terminal_ring: Arc<Mutex<TerminalRing>>,
1984    spawn_events: SpawnEventFeed,
1985    child_roster: ChildRoster,
1986    #[cfg(target_os = "linux")]
1987    cgroup_placement: Option<subc_cgroup::Placement>,
1988    #[cfg(test)]
1989    test_seed_stale_facts_before_enable_spawn: bool,
1990    #[cfg(test)]
1991    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1992}
1993
1994impl SupervisorRuntimeConfig {
1995    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1996        FailureContext {
1997            daemon_initiated,
1998            shutdown: self.child_roster.shutdown_flag(),
1999        }
2000    }
2001}
2002
2003#[derive(Debug, Clone, PartialEq, Eq)]
2004struct SupervisedConfiguration {
2005    spec: ModuleSpec,
2006    health: HealthConfig,
2007}
2008
2009/// Shared daemon lookup table for supervised module handles.
2010///
2011/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2012/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2013/// launch nonces recorded at spawn are checked by the same daemon instance.
2014#[derive(Debug, Clone, Default)]
2015pub struct SupervisorHandle {
2016    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2017    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2018    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2019    /// Module ids the supervisor has taken on. An id is added BEFORE the
2020    /// module's first process is spawned and removed only when the module
2021    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2022    /// the keys of `modules`.
2023    ///
2024    /// `modules` cannot answer "is this module configured?" on its own: a
2025    /// [`SupervisedModule`] only exists once its process has been spawned, and
2026    /// a fast child can connect, register, sync its scopes and ask about them
2027    /// before the supervisor has inserted it. Answering "not configured" in that
2028    /// gap makes scope admission refuse with the terminal "will never sync"
2029    /// instead of the retryable "has not synced yet".
2030    configured_ids: Arc<Mutex<HashSet<String>>>,
2031    spawn_events: SpawnEventFeed,
2032    /// The current expected launch nonce for each reserved module_id. Set when the
2033    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2034    /// non-reserved module never has an entry here and is never nonce-checked.
2035    /// Reserved module ids and the nonce that authorizes their next HELLO.
2036    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2037    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2038    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2039    /// had NO entry and admitted anyone: the reservation protected the nonce
2040    /// holder, not the NAME (found live by CKCRED's canary probe registering
2041    /// against a reserved scratch id).
2042    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2043    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2044    ///
2045    /// This is deliberately in-memory only: subc is state-free across daemon
2046    /// restarts, and the tombstone only explains the hours-after-removal window
2047    /// while this executing daemon is still alive. Do not persist it in a store.
2048    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2049    /// The current launch nonce for every supervised spawn. This is separate from
2050    /// reserved_nonces because consumer route.open attestation applies to all spawned
2051    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2052    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2053    /// Reserved namespace prefixes mapped to the supervised owner module whose
2054    /// current spawn nonce authorizes HELLO claims below the prefix.
2055    ///
2056    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2057    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2058    /// accidental collisions and lower-trust processes from squatting protected
2059    /// namespaces.
2060    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2061    /// Blue/green swaps in progress, by module id. An entry exists from just
2062    /// before the candidate process is spawned until the swap has failed, or
2063    /// has cut over and the old process is gone. While it exists, HELLO for the
2064    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2065    /// consumer attestation accepts both processes' nonces.
2066    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2067    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2068    promotion_observer: PromotionObserverSlot,
2069    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2070    /// this daemon-wide ordering, a rescan could retire or update a module while a
2071    /// concurrent reload still held its old handle and launch specification.
2072    operation_lock: Arc<AsyncMutex<()>>,
2073}
2074
2075/// Told when a swap has promoted its candidate to be the module's active
2076/// registration.
2077///
2078/// An ordinary HELLO runs the control plane's registration side effects (the
2079/// capability cache, the deny census, the requirement recompute) as it
2080/// registers. A swap candidate's HELLO does not, because it is not routable;
2081/// promotion is when those must run instead, and promotion happens in the
2082/// supervisor, which has no other way into the control handler.
2083pub(crate) trait SwapPromotionObserver: Send + Sync {
2084    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2085}
2086
2087/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2088/// control handler) owns this handle, so a strong reference back would be a
2089/// cycle that keeps both alive.
2090#[derive(Clone, Default)]
2091struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2092
2093impl fmt::Debug for PromotionObserverSlot {
2094    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2095        f.write_str("PromotionObserverSlot")
2096    }
2097}
2098
2099/// The nonces of one open swap.
2100#[derive(Debug, Clone)]
2101struct OpenSwap {
2102    /// The launch nonce minted for the candidate process. It is the swap
2103    /// token: the only thing that admits a HELLO into the candidate slot.
2104    candidate_nonce: String,
2105    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2106    /// here because cutover moves the module's recorded spawn nonce to the
2107    /// candidate while the incumbent is still draining and its consumers are
2108    /// still attesting with this one.
2109    incumbent_nonce: Option<String>,
2110    /// Set once a HELLO has been admitted with the swap token, so the token
2111    /// admits one registration and cannot be replayed after cutover empties
2112    /// the candidate slot.
2113    candidate_admitted: bool,
2114}
2115
2116/// What the swap gate says about a HELLO. See
2117/// [`SupervisorHandle::swap_hello_admission`].
2118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2119pub(crate) enum SwapHelloAdmission {
2120    /// No swap is open for the id (or the HELLO carries the incumbent's own
2121    /// nonce); the ordinary gates decide.
2122    NotSwapping,
2123    /// The HELLO carries the swap token: register it into the candidate slot.
2124    Candidate,
2125    /// A swap is open and the HELLO carries a nonce the supervisor did not
2126    /// mint for this id, no nonce, or a token already used.
2127    Refused,
2128}
2129
2130#[derive(Debug, Clone, PartialEq, Eq)]
2131pub(crate) enum ReservedHelloRejection {
2132    Exact {
2133        module_id: String,
2134    },
2135    Prefix {
2136        prefix: String,
2137        owner_module_id: String,
2138    },
2139}
2140
2141impl SupervisorHandle {
2142    pub fn new() -> Self {
2143        Self::default()
2144    }
2145
2146    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2147        self.spawn_events.snapshot()
2148    }
2149
2150    pub(crate) fn subscribe_spawns(
2151        &self,
2152        connection_id: ConnectionId,
2153        corr: u64,
2154        version: u8,
2155        since: Option<SpawnCursor>,
2156        sink: FrameSink,
2157    ) -> Result<(), SpawnSubscribeRefusal> {
2158        self.spawn_events
2159            .subscribe(connection_id, corr, version, since, sink)
2160    }
2161
2162    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2163        self.spawn_events.cancel(connection_id, corr)
2164    }
2165
2166    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2167        self.spawn_events.remove_connection(connection_id);
2168    }
2169
2170    #[cfg(any(test, feature = "test-support"))]
2171    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2172        assert!(capacity > 0, "spawn event capacity must be non-zero");
2173        self.spawn_events.set_capacity(capacity);
2174    }
2175
2176    #[cfg(any(test, feature = "test-support"))]
2177    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2178        self.spawn_events.subscriber_count()
2179    }
2180
2181    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2182    /// a respawn invalidates stale consumer identities.
2183    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2184        self.spawn_nonces
2185            .lock()
2186            .unwrap_or_else(|poisoned| poisoned.into_inner())
2187            .insert(module_id.to_string(), nonce);
2188    }
2189
2190    /// Record the launch nonce expected from the next HELLO for a reserved module,
2191    /// replacing any prior nonce (a respawn invalidates the previous one).
2192    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2193        self.reserved_nonces
2194            .lock()
2195            .unwrap_or_else(|poisoned| poisoned.into_inner())
2196            .insert(module_id.to_string(), Some(nonce));
2197    }
2198
2199    /// Record namespace prefixes owned by a supervised module.
2200    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2201        let mut owners = self
2202            .reserved_prefix_owners
2203            .lock()
2204            .unwrap_or_else(|poisoned| poisoned.into_inner());
2205        owners.retain(|_, owner| owner != owner_module_id);
2206        for prefix in prefixes {
2207            owners.insert(prefix.clone(), owner_module_id.to_string());
2208        }
2209    }
2210
2211    /// The launch nonce most recently minted for a module's spawn, if any.
2212    #[cfg(test)]
2213    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2214        self.spawn_nonces
2215            .lock()
2216            .unwrap_or_else(|poisoned| poisoned.into_inner())
2217            .get(module_id)
2218            .cloned()
2219    }
2220
2221    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2222        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2223        let spawn_nonce = self
2224            .spawn_nonces
2225            .lock()
2226            .unwrap_or_else(|poisoned| poisoned.into_inner())
2227            .get(&spec.module_id)
2228            .cloned();
2229        let mut reserved_nonces = self
2230            .reserved_nonces
2231            .lock()
2232            .unwrap_or_else(|poisoned| poisoned.into_inner());
2233        if spec.reserved {
2234            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2235            // reserved name whose module has never spawned has no legitimate
2236            // holder, and the entry's absence is what used to leave the name
2237            // open to the first claimant.
2238            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2239        }
2240        drop(reserved_nonces);
2241        // A later unreserved declaration must not silently unreserve an id that
2242        // was retained after its reserved configuration was removed. The explicit
2243        // release ceremony is the only operation that retires that gate.
2244        self.removal_tombstones
2245            .lock()
2246            .unwrap_or_else(|poisoned| poisoned.into_inner())
2247            .remove(&spec.module_id);
2248    }
2249
2250    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2251    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2252    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2253    /// with no matching prefix are always authorized.
2254    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2255        self.reserved_hello_verification(module_id, presented)
2256            .is_ok()
2257    }
2258
2259    /// Whether HELLO passed a nonce check that was actually required by reserved
2260    /// module policy, or the refusal explaining why a required check failed.
2261    pub(crate) fn reserved_hello_verification(
2262        &self,
2263        module_id: &str,
2264        presented: Option<&str>,
2265    ) -> Result<bool, ReservedHelloRejection> {
2266        let nonces = self
2267            .reserved_nonces
2268            .lock()
2269            .unwrap_or_else(|poisoned| poisoned.into_inner());
2270        if let Some(expected) = nonces.get(module_id) {
2271            // `None` = reserved with no legitimate holder: refuse every
2272            // presentation, because no process can hold a nonce that was never
2273            // minted. Only a real minted nonce admits, in constant time.
2274            let authorized = match expected {
2275                Some(expected) => {
2276                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2277                }
2278                None => false,
2279            };
2280            if authorized {
2281                return Ok(true);
2282            }
2283            return Err(ReservedHelloRejection::Exact {
2284                module_id: module_id.to_string(),
2285            });
2286        }
2287        drop(nonces);
2288
2289        let matched_prefix = self
2290            .reserved_prefix_owners
2291            .lock()
2292            .unwrap_or_else(|poisoned| poisoned.into_inner())
2293            .iter()
2294            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2295            .max_by_key(|(prefix, _)| prefix.len())
2296            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2297        let Some((prefix, owner_module_id)) = matched_prefix else {
2298            return Ok(false);
2299        };
2300
2301        let authorized = presented.is_some_and(|presented| {
2302            self.spawn_nonces
2303                .lock()
2304                .unwrap_or_else(|poisoned| poisoned.into_inner())
2305                .get(&owner_module_id)
2306                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2307                // While the owner is being swapped, children started by
2308                // either of its two processes hold that process's nonce.
2309                || self.swap_nonce_matches(&owner_module_id, presented)
2310        });
2311        if authorized {
2312            Ok(true)
2313        } else {
2314            Err(ReservedHelloRejection::Prefix {
2315                prefix,
2316                owner_module_id,
2317            })
2318        }
2319    }
2320
2321    /// Whether a consumer connection proved it came from a daemon-spawned module.
2322    ///
2323    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2324    /// accepted only for module ids the supervisor has spawned.
2325    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2326        if presented.is_empty() {
2327            return false;
2328        }
2329        let nonces = self
2330            .spawn_nonces
2331            .lock()
2332            .unwrap_or_else(|poisoned| poisoned.into_inner());
2333        let current = nonces
2334            .get(module_id)
2335            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2336        drop(nonces);
2337        // During a swap two processes of the module are alive, and a consumer
2338        // started by either one presents that process's nonce. Accepting only
2339        // the recorded one would fail the incumbent's consumers for the whole
2340        // overlap once cutover moves the record to the candidate.
2341        current || self.swap_nonce_matches(module_id, presented)
2342    }
2343
2344    /// Whether `presented` is either nonce of an open swap for `module_id`.
2345    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2346        let swaps = self
2347            .swaps
2348            .lock()
2349            .unwrap_or_else(|poisoned| poisoned.into_inner());
2350        swaps.get(module_id).is_some_and(|swap| {
2351            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2352                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2353                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2354                })
2355        })
2356    }
2357
2358    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2359    /// Called before the candidate process exists.
2360    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2361        let incumbent_nonce = self
2362            .spawn_nonces
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner())
2365            .get(module_id)
2366            .cloned();
2367        self.swaps
2368            .lock()
2369            .unwrap_or_else(|poisoned| poisoned.into_inner())
2370            .insert(
2371                module_id.to_string(),
2372                OpenSwap {
2373                    candidate_nonce,
2374                    incumbent_nonce,
2375                    candidate_admitted: false,
2376                },
2377            );
2378    }
2379
2380    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2381    /// the module's recorded one.
2382    pub(crate) fn close_swap(&self, module_id: &str) {
2383        self.swaps
2384            .lock()
2385            .unwrap_or_else(|poisoned| poisoned.into_inner())
2386            .remove(module_id);
2387    }
2388
2389    /// Install the observer told about swap promotions, replacing any earlier
2390    /// one.
2391    pub(crate) fn set_swap_promotion_observer(
2392        &self,
2393        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2394    ) {
2395        *self
2396            .promotion_observer
2397            .0
2398            .lock()
2399            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2400    }
2401
2402    /// Tell the installed observer, if it is still alive, that a swap promoted
2403    /// `registration`.
2404    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2405        let observer = self
2406            .promotion_observer
2407            .0
2408            .lock()
2409            .unwrap_or_else(|poisoned| poisoned.into_inner())
2410            .as_ref()
2411            .and_then(std::sync::Weak::upgrade);
2412        if let Some(observer) = observer {
2413            observer.swap_promoted(registration);
2414        }
2415    }
2416
2417    /// Whether a swap is open for `module_id`.
2418    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2419        self.swaps
2420            .lock()
2421            .unwrap_or_else(|poisoned| poisoned.into_inner())
2422            .contains_key(module_id)
2423    }
2424
2425    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2426    /// respawn would, once cutover has made the candidate the module's process.
2427    /// The swap stays open so the incumbent's nonce keeps attesting until the
2428    /// incumbent has drained and exited.
2429    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2430        let candidate_nonce = self
2431            .swaps
2432            .lock()
2433            .unwrap_or_else(|poisoned| poisoned.into_inner())
2434            .get(module_id)
2435            .map(|swap| swap.candidate_nonce.clone());
2436        let Some(nonce) = candidate_nonce else {
2437            return;
2438        };
2439        self.set_spawn_nonce(module_id, nonce.clone());
2440        if reserved {
2441            self.set_reserved_nonce(module_id, nonce);
2442        }
2443    }
2444
2445    /// The swap gate for a HELLO claiming `module_id`.
2446    ///
2447    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2448    /// presents the candidate nonce, which the reserved gate (holding the
2449    /// incumbent's nonce) would refuse as `reserved_module` before swap
2450    /// admission was ever reached. And it applies to unreserved ids too: for an
2451    /// unreserved id the only thing that ever stopped a second process claiming
2452    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2453    /// refusal a swap lifts for its candidate.
2454    ///
2455    /// The incumbent's own nonce falls through to the ordinary gates, which
2456    /// treat it as they always have (a live incumbent is refused as a
2457    /// duplicate). Anything else while a swap is open is refused, including an
2458    /// absent nonce.
2459    pub(crate) fn swap_hello_admission(
2460        &self,
2461        module_id: &str,
2462        presented: Option<&str>,
2463    ) -> SwapHelloAdmission {
2464        let swaps = self
2465            .swaps
2466            .lock()
2467            .unwrap_or_else(|poisoned| poisoned.into_inner());
2468        let Some(swap) = swaps.get(module_id) else {
2469            return SwapHelloAdmission::NotSwapping;
2470        };
2471        let Some(presented) = presented else {
2472            return SwapHelloAdmission::Refused;
2473        };
2474        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2475            return if swap.candidate_admitted {
2476                SwapHelloAdmission::Refused
2477            } else {
2478                SwapHelloAdmission::Candidate
2479            };
2480        }
2481        if swap
2482            .incumbent_nonce
2483            .as_deref()
2484            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2485        {
2486            return SwapHelloAdmission::NotSwapping;
2487        }
2488        SwapHelloAdmission::Refused
2489    }
2490
2491    /// Record that the swap token has registered a candidate, so it admits no
2492    /// second HELLO.
2493    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2494        if let Some(swap) = self
2495            .swaps
2496            .lock()
2497            .unwrap_or_else(|poisoned| poisoned.into_inner())
2498            .get_mut(module_id)
2499        {
2500            swap.candidate_admitted = true;
2501        }
2502    }
2503
2504    /// Test/support lookup for the current launch nonce of a supervised spawn.
2505    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2506        self.spawn_nonces
2507            .lock()
2508            .unwrap_or_else(|poisoned| poisoned.into_inner())
2509            .get(module_id)
2510            .cloned()
2511    }
2512
2513    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2514    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2515        self.reserved_nonces
2516            .lock()
2517            .unwrap_or_else(|poisoned| poisoned.into_inner())
2518            .get(module_id)
2519            .cloned()
2520            .flatten()
2521    }
2522
2523    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2524        // Normally already marked before the process was spawned; marking here
2525        // too keeps `configured_ids` a superset of the roster for any caller
2526        // that inserts a module directly.
2527        self.mark_configured(module.module_id());
2528        let mut modules = self
2529            .modules
2530            .lock()
2531            .unwrap_or_else(|poisoned| poisoned.into_inner());
2532        modules.insert(module.module_id().to_string(), module)
2533    }
2534
2535    /// Record that the supervisor has taken on `module_id`. Called before the
2536    /// module's first process is spawned, so that by the time that process can
2537    /// register, [`Self::is_configured`] already answers true.
2538    fn mark_configured(&self, module_id: &str) {
2539        self.configured_ids
2540            .lock()
2541            .unwrap_or_else(|poisoned| poisoned.into_inner())
2542            .insert(module_id.to_string());
2543    }
2544
2545    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2546    /// before it was ever put on the roster. A module already on the roster
2547    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2548    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2549        let modules = self
2550            .modules
2551            .lock()
2552            .unwrap_or_else(|poisoned| poisoned.into_inner());
2553        if !modules.contains_key(module_id) {
2554            self.configured_ids
2555                .lock()
2556                .unwrap_or_else(|poisoned| poisoned.into_inner())
2557                .remove(module_id);
2558        }
2559    }
2560
2561    /// Whether `module_id` is a module this daemon supervises: on the roster,
2562    /// or about to be (its process is being spawned right now).
2563    ///
2564    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2565    /// for scopes: a supervised module's process can register and sync before
2566    /// [`Self::get`] can return it, and in that window it is still a module
2567    /// that will sync, not one that never will.
2568    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2569        self.configured_ids
2570            .lock()
2571            .unwrap_or_else(|poisoned| poisoned.into_inner())
2572            .contains(module_id)
2573    }
2574
2575    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2576        let modules = self
2577            .modules
2578            .lock()
2579            .unwrap_or_else(|poisoned| poisoned.into_inner());
2580        modules.get(module_id).cloned()
2581    }
2582
2583    pub(crate) fn record_late_health_answer(
2584        &self,
2585        module_id: &str,
2586        latency_ms: u64,
2587    ) -> Result<bool, SuperviseError> {
2588        let Some(module) = self.get(module_id) else {
2589            return Ok(false);
2590        };
2591        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2592            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2593            state.health.last_late_answer_latency_ms = Some(latency_ms);
2594            // A late answer is an answer: the module served the probe, just past
2595            // the deadline. Leaving the miss streak in place while logging
2596            // "proves the module is alive" is how a CPU-starved module that
2597            // answers every probe a few seconds late still marches to the
2598            // threshold and gets killed — the exact kill class `NoAnswer` is
2599            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2600            // is degradation, and degradation reports; it does not restart.
2601            state.health.consecutive_failures = 0;
2602        })?;
2603        Ok(true)
2604    }
2605
2606    /// Arm the one-shot marker for the module process that this caller
2607    /// deliberately initiated severance against. Generic connection teardown
2608    /// must not call this:
2609    /// a surviving process would otherwise retain an exemption for a later
2610    /// genuine crash.
2611    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2612        let Some(module) = self.get(module_id) else {
2613            return Ok(false);
2614        };
2615        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2616        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2617            return Ok(false);
2618        };
2619        drop(snapshot);
2620        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2621    }
2622
2623    pub fn list(&self) -> Vec<SupervisedModule> {
2624        let modules = self
2625            .modules
2626            .lock()
2627            .unwrap_or_else(|poisoned| poisoned.into_inner());
2628        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2629        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2630        modules
2631    }
2632
2633    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2634        self.activity_images
2635            .lock()
2636            .unwrap_or_else(|p| p.into_inner())
2637            .remove(module_id);
2638        self.spawn_nonces
2639            .lock()
2640            .unwrap_or_else(|poisoned| poisoned.into_inner())
2641            .remove(module_id);
2642        self.close_swap(module_id);
2643        let mut reserved_nonces = self
2644            .reserved_nonces
2645            .lock()
2646            .unwrap_or_else(|poisoned| poisoned.into_inner());
2647        if reserved_nonces.contains_key(module_id) {
2648            // The old nonce must die with the removed process, but the exact-id
2649            // gate remains until an operator explicitly releases it.
2650            reserved_nonces.insert(module_id.to_string(), None);
2651        }
2652        drop(reserved_nonces);
2653        self.reserved_prefix_owners
2654            .lock()
2655            .unwrap_or_else(|poisoned| poisoned.into_inner())
2656            .retain(|_, owner| owner != module_id);
2657        let removed = self
2658            .modules
2659            .lock()
2660            .unwrap_or_else(|poisoned| poisoned.into_inner())
2661            .remove(module_id);
2662        self.configured_ids
2663            .lock()
2664            .unwrap_or_else(|poisoned| poisoned.into_inner())
2665            .remove(module_id);
2666        removed
2667    }
2668
2669    /// Remember a module removed by a non-preview rescan so route.open can
2670    /// distinguish that intentional removal from an unknown id.
2671    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2672        self.removal_tombstones
2673            .lock()
2674            .unwrap_or_else(|poisoned| poisoned.into_inner())
2675            .insert(module_id.to_string(), unix_ms_now());
2676    }
2677
2678    /// Return how long ago a rescan removed this module in milliseconds.
2679    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2680        self.removal_tombstones
2681            .lock()
2682            .unwrap_or_else(|poisoned| poisoned.into_inner())
2683            .get(module_id)
2684            .copied()
2685            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2686    }
2687
2688    /// Retire a reserved-id gate only after its module has left supervision.
2689    ///
2690    /// A retained gate has no live nonce (`None`), so releasing any other entry
2691    /// would weaken a currently configured or otherwise active reservation.
2692    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2693        if self.get(module_id).is_some() {
2694            return false;
2695        }
2696        let mut reserved_nonces = self
2697            .reserved_nonces
2698            .lock()
2699            .unwrap_or_else(|poisoned| poisoned.into_inner());
2700        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2701            return false;
2702        }
2703        reserved_nonces.remove(module_id);
2704        true
2705    }
2706
2707    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2708        Arc::clone(&self.operation_lock)
2709    }
2710}
2711
2712/// Process supervisor for subc-owned singleton modules.
2713#[derive(Debug, Clone)]
2714pub struct Supervisor {
2715    registry: Arc<Registry>,
2716    restart_policy: RestartPolicy,
2717    drain_timeout: Duration,
2718    connection_file_path: Option<PathBuf>,
2719    capture_logs_dir: Option<PathBuf>,
2720    forwarding: Option<Arc<ForwardingTable>>,
2721    process_liveness: Arc<SupervisorProcessLiveness>,
2722    supervisor_handle: Option<SupervisorHandle>,
2723    health: HealthConfig,
2724    daemon_start_clock: crate::clock::StartClock,
2725    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2726    spawn_events: SpawnEventFeed,
2727    provenance_probe: ExecutableIdentityProbe,
2728    /// Every process spawned through this supervisor (and its clones) and not
2729    /// yet reaped, so daemon shutdown can end them.
2730    child_roster: ChildRoster,
2731    #[cfg(target_os = "linux")]
2732    cgroup_placement: Option<subc_cgroup::Placement>,
2733    #[cfg(test)]
2734    test_after_first_spawn: AfterFirstSpawnHook,
2735}
2736
2737/// A roster entry whose monitor and first process have not started yet.
2738/// Keeping the command receiver here lets discovery serve status immediately
2739/// while start/stop requests wait until bootstrap has published the connection file.
2740pub(crate) struct RegisteredConfiguredModule {
2741    spec: ModuleSpec,
2742    runtime: SupervisorRuntimeConfig,
2743    module: SupervisedModule,
2744    commands: mpsc::Receiver<SupervisorCommand>,
2745}
2746
2747/// Test-only hook run on the path that takes on a new module, right after its
2748/// first `spawn_child` returns (the process exists and could already be
2749/// registering) and before that process is handed to the module's supervise
2750/// loop and put on the roster. Lets a test observe what a fast child would see
2751/// in that window without racing a real one.
2752#[cfg(test)]
2753#[derive(Clone, Default)]
2754struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2755
2756#[cfg(test)]
2757type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2758
2759#[cfg(test)]
2760impl fmt::Debug for AfterFirstSpawnHook {
2761    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2762        f.write_str("AfterFirstSpawnHook")
2763    }
2764}
2765
2766#[cfg(test)]
2767impl AfterFirstSpawnHook {
2768    fn run(&self, module_id: &str) {
2769        if let Some(hook) = &self.0 {
2770            hook(module_id);
2771        }
2772    }
2773}
2774
2775impl Supervisor {
2776    #[cfg(test)]
2777    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2778        let supervisor = Self::new(registry, policy);
2779        #[cfg(target_os = "macos")]
2780        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2781        supervisor
2782    }
2783    /// Verify the trampoline once when configured. Missing private OS support
2784    /// refuses every macOS launch by name but does not stop the daemon's control
2785    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2786    /// entry point; the library must not exec an arbitrary hosting program.
2787    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2788        let path = path.into();
2789        #[cfg(target_os = "macos")]
2790        {
2791            let result = probe_privacy_trampoline(&path).map(|()| path);
2792            if let Err(cause) = &result {
2793                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2794            }
2795            self.child_roster.set_privacy_trampoline(result);
2796        }
2797        #[cfg(not(target_os = "macos"))]
2798        let _ = path;
2799        self
2800    }
2801    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2802    /// capability-requirement log to read.
2803    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2804        self.child_roster.shutdown_flag()
2805    }
2806
2807    /// The first step of an announced daemon shutdown, before the notice and
2808    /// before any connection is closed.
2809    ///
2810    /// Sets the daemon-shutdown flag first: from here on no module is
2811    /// respawned (crash restart, operator restart, or swap), and every child
2812    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2813    /// the module exits on the EOF this shutdown gives it or is signalled by a
2814    /// service manager that kills the whole cgroup. Then writes the journal's
2815    /// shutdown marker, which records the instant and closes this daemon
2816    /// incarnation's stretch of the journal.
2817    #[cfg(unix)]
2818    pub(crate) fn begin_daemon_shutdown(&self) {
2819        self.child_roster.close();
2820        if let Some(journal) = &self.terminal_journal {
2821            journal.stamp_shutdown();
2822        }
2823    }
2824
2825    /// Announce a cut while established connections can still carry replies.
2826    /// These budgets promise notice and a bounded wait, not child completion;
2827    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2828    #[cfg(unix)]
2829    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2830        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2831        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2832        let Some(forwarding) = &self.forwarding else {
2833            return Ok(());
2834        };
2835        let notice_started = Instant::now();
2836        let modules = forwarding
2837            .begin_daemon_drain()
2838            .map_err(SuperviseError::Forwarding)?;
2839        let deadline_ms =
2840            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2841        let mut notices = tokio::task::JoinSet::new();
2842        let mut drains = Vec::new();
2843        for (module_id, endpoint) in modules {
2844            let Some(target) = forwarding
2845                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2846                .map_err(SuperviseError::Forwarding)?
2847            else {
2848                continue;
2849            };
2850            let routes = forwarding
2851                .endpoint_routes(target.endpoint)
2852                .map_err(SuperviseError::Forwarding)?;
2853            // Restart allows deployed consumers to reopen after the new daemon
2854            // appears. The wire reason stays `restart`; what tells a daemon cut
2855            // apart from a module restart afterwards is the terminal record
2856            // itself, whose disposition is `daemon_shutdown` for every exit
2857            // observed once `begin_daemon_shutdown` has run.
2858            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2859                reason: RouteCloseReason::Restart,
2860                deadline_ms,
2861            })
2862            .expect("module draining serializes");
2863            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2864            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2865            for route in routes {
2866                let client = route.goodbye_target;
2867                if let Some((_, channels)) = clients
2868                    .iter_mut()
2869                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2870                {
2871                    channels.push(client.channel);
2872                } else {
2873                    let channel = client.channel;
2874                    clients.push((client, vec![channel]));
2875                }
2876            }
2877            for (client, mut channels) in clients {
2878                channels.sort_unstable();
2879                channels.dedup();
2880                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2881                    module_id: module_id.clone(),
2882                    channels,
2883                    reason: RouteCloseReason::Restart,
2884                })
2885                .expect("route closing serializes");
2886                recipients.push((client.sink, client.negotiated_ver, closing));
2887            }
2888            for (sink, version, body) in recipients {
2889                notices.spawn(async move {
2890                    let frame = Frame::build_with_version(
2891                        version,
2892                        FrameType::Push,
2893                        control_flags(),
2894                        0,
2895                        0,
2896                        0,
2897                        body,
2898                    )
2899                    .expect("bounded lifecycle notice frame builds");
2900                    sink.send_flushed(frame).await
2901                });
2902            }
2903            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2904            drains.push((module_id, target.endpoint, gauges));
2905        }
2906        // A quiet forwarding table is not proof that queued notices reached the
2907        // socket. Wait for writer flush acknowledgements before testing quiescence.
2908        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2909        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2910            if !matches!(result, Ok(Ok(()))) {
2911                warn!(?result, "daemon shutdown notice delivery failed");
2912            }
2913        }
2914        notices.abort_all();
2915        let notice_elapsed = notice_started.elapsed();
2916        let drain_started = Instant::now();
2917        let deadline = drain_started + DRAIN_BUDGET;
2918        let mut waits = tokio::task::JoinSet::new();
2919        for (module_id, endpoint, gauges) in drains {
2920            let forwarding = Arc::clone(forwarding);
2921            let mut runtime = self.runtime_config();
2922            runtime.health.cadence = Duration::from_millis(100);
2923            waits.spawn(async move {
2924                let wait_result = wait_for_forwarding_quiescence(
2925                    &forwarding,
2926                    &module_id,
2927                    &runtime,
2928                    endpoint,
2929                    deadline,
2930                    &gauges,
2931                    DrainScope::Endpoint(endpoint),
2932                    &mut DrainGaugeTrace::new(&gauges),
2933                )
2934                .await;
2935                // Store the measurement before the task returns: quiescence
2936                // may be reached just as the deadline stops the join loop from
2937                // collecting results. Unfinished or failed waits report false.
2938                forwarding
2939                    .record_daemon_drain_result(
2940                        endpoint,
2941                        drained_after_quiescence_wait(&wait_result),
2942                    )
2943                    .map_err(SuperviseError::Forwarding)?;
2944                wait_result
2945            });
2946        }
2947        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2948            if !matches!(result, Ok(Ok(true))) {
2949                warn!(?result, "daemon shutdown drain did not reach quiescence");
2950            }
2951        }
2952        // Measure only the forwarding drain, not child teardown or the caller's
2953        // signal delivery and process-reaping delays.
2954        info!(
2955            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2956            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2957            "daemon shutdown forwarding drain complete"
2958        );
2959        Ok(())
2960    }
2961
2962    /// The last step of an announced daemon shutdown, after the notice and the
2963    /// drain: send every registered module a module GOODBYE, the same planned
2964    /// stop signal `ck module stop` gives, then close every connection so each
2965    /// subc module sees EOF and starts its own teardown, then end every
2966    /// supervised child that has not exited
2967    /// by its own deadline (its drain budget, capped). Modules lead their own
2968    /// process groups, so a
2969    /// service manager's group kill no longer reaches them; without this a
2970    /// child that does not stop on EOF (every `protocol: "none"` child, which
2971    /// has no connection) would outlive the daemon. Every wait is bounded (see
2972    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2973    #[cfg(unix)]
2974    pub(crate) async fn end_children_for_daemon_shutdown(
2975        &self,
2976        already_escalated: bool,
2977        escalate: impl std::future::Future<Output = ()>,
2978    ) {
2979        tokio::pin!(escalate);
2980        let mut escalated = already_escalated;
2981        if let Some(forwarding) = &self.forwarding {
2982            let reason = CloseReason::new(
2983                "daemon_shutdown",
2984                "the daemon is exiting after its shutdown notice and drain",
2985            );
2986            if escalated {
2987                // The operator asked to stop waiting: queue the GOODBYEs but
2988                // do not wait for them to be written.
2989                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2990            } else {
2991                tokio::select! {
2992                    biased;
2993                    _ = escalate.as_mut() => {
2994                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2995                        escalated = true;
2996                    }
2997                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2998                }
2999            }
3000            let closed = forwarding.close_all_connections(&reason);
3001            debug!(closed, "closed established connections for daemon shutdown");
3002        }
3003        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3004        // already completed and must not be polled again; the child shutdown
3005        // wait is told it is escalated and gets a future that never fires.
3006        let escalated_here = escalated && !already_escalated;
3007        let remaining_escalate = async move {
3008            if escalated_here {
3009                std::future::pending::<()>().await;
3010            } else {
3011                escalate.await;
3012            }
3013        };
3014        crate::child_roster::end_children_for_daemon_shutdown(
3015            &self.child_roster,
3016            escalated,
3017            remaining_escalate,
3018        )
3019        .await;
3020    }
3021
3022    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3023        Self {
3024            registry,
3025            restart_policy,
3026            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3027            connection_file_path: None,
3028            capture_logs_dir: None,
3029            forwarding: None,
3030            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3031            supervisor_handle: None,
3032            health: HealthConfig::default(),
3033            daemon_start_clock: crate::clock::StartClock::capture(),
3034            terminal_journal: None,
3035            spawn_events: SpawnEventFeed::default(),
3036            provenance_probe: ExecutableIdentityProbe::default(),
3037            child_roster: ChildRoster::default(),
3038            #[cfg(target_os = "linux")]
3039            cgroup_placement: None,
3040            #[cfg(test)]
3041            test_after_first_spawn: AfterFirstSpawnHook::default(),
3042        }
3043    }
3044
3045    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3046        self.drain_timeout = drain_timeout;
3047        self
3048    }
3049
3050    /// The drain budget a module gets when its config entry sets none.
3051    pub(crate) fn default_drain_timeout(&self) -> Duration {
3052        self.drain_timeout
3053    }
3054
3055    pub fn with_process_liveness(
3056        mut self,
3057        process_liveness: Arc<SupervisorProcessLiveness>,
3058    ) -> Self {
3059        self.process_liveness = process_liveness;
3060        self
3061    }
3062
3063    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3064        self.connection_file_path = Some(connection_file_path.into());
3065        self
3066    }
3067
3068    /// Enables daemon-owned capture files for supervised stdout and stderr.
3069    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3070        self.capture_logs_dir = Some(logs_dir.into());
3071        self
3072    }
3073
3074    /// Names this daemon lifetime in spawn events, independently of whether a
3075    /// terminal journal is configured.
3076    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3077        // A millisecond start stamp can repeat after clock rollback or a rapid
3078        // restart. Use the connection file's random daemon_id instead: it already
3079        // identifies this daemon lifetime independently of the wall clock.
3080        self.spawn_events.configure_incarnation(daemon_incarnation);
3081        self
3082    }
3083
3084    /// Enables best-effort history shared by every supervised module. Without
3085    /// it, terminal history is kept only in each module's in-memory ring.
3086    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3087        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3088        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3089            path,
3090            daemon_incarnation,
3091        )));
3092        this
3093    }
3094
3095    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3096        self.forwarding = Some(forwarding);
3097        self
3098    }
3099
3100    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3101        self.spawn_events = supervisor_handle.spawn_events.clone();
3102        self.supervisor_handle = Some(supervisor_handle);
3103        self
3104    }
3105
3106    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3107        self.health = health;
3108        self
3109    }
3110
3111    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3112    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3113    /// record is kept.
3114    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3115        self.child_roster.record_to(path.into());
3116        self
3117    }
3118
3119    #[cfg(target_os = "linux")]
3120    pub fn with_cgroup_placement(
3121        mut self,
3122        cgroup_placement: Option<subc_cgroup::Placement>,
3123    ) -> Self {
3124        self.cgroup_placement = cgroup_placement;
3125        self
3126    }
3127
3128    /// Spawn `spec.program` and start monitoring it.
3129    ///
3130    /// The child is expected to parse `--subc <connection-file-path>`, read the
3131    /// TCP+key connection file, authenticate to the already-running listener, and
3132    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3133    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3134        validate_spec(&spec)?;
3135        self.establish_identity(&spec);
3136
3137        let runtime = self.runtime_config();
3138        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3139        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3140        let registration_watch = watch_registration(&self.registry, &spec);
3141        let spawned = spawn_child(
3142            &spec,
3143            runtime.connection_file_path.as_deref(),
3144            self.supervisor_handle.as_ref(),
3145            &runtime.stderr_ring,
3146            runtime.capture_logs_dir.as_deref(),
3147            &runtime.child_roster,
3148            #[cfg(target_os = "linux")]
3149            runtime.cgroup_placement.as_ref(),
3150        );
3151        #[cfg(test)]
3152        self.test_after_first_spawn.run(&spec.module_id);
3153        let mut child = match spawned {
3154            Ok(child) => child,
3155            Err(err) => {
3156                // Unlike the configured paths, a failed `spawn` leaves nothing
3157                // on the roster, so the module must not stay marked configured.
3158                self.abandon_unrostered(&spec.module_id);
3159                return Err(err);
3160            }
3161        };
3162        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3163        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3164        if let Some(spawn) = activity_spawn {
3165            spawn.complete(&snapshot);
3166        }
3167
3168        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3169    }
3170
3171    /// Make `spec`'s module count as configured, with its identity gates
3172    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3173    /// exists.
3174    ///
3175    /// Every path that takes on a new module calls this before `spawn_child`.
3176    /// The order is the point: the child can connect, register, sync its
3177    /// scopes and ask about them as soon as it is spawned, and the module is
3178    /// only put on the roster after `spawn_child` returns. Were the mark set
3179    /// with the roster entry, a fast child would see its own owner reported
3180    /// as not configured, and a scoped `route.open` in that window would be
3181    /// refused as terminal `scope_not_live` ("will never sync") instead of
3182    /// retryable `scope_not_synced`.
3183    fn establish_identity(&self, spec: &ModuleSpec) {
3184        if let Some(supervisor_handle) = &self.supervisor_handle {
3185            supervisor_handle.apply_identity_configuration(spec);
3186            supervisor_handle.mark_configured(&spec.module_id);
3187        }
3188    }
3189
3190    /// Take back [`Self::establish_identity`]'s configured mark when the
3191    /// module will not be put on the roster after all.
3192    fn abandon_unrostered(&self, module_id: &str) {
3193        if let Some(supervisor_handle) = &self.supervisor_handle {
3194            supervisor_handle.unmark_configured_unless_rostered(module_id);
3195        }
3196    }
3197
3198    /// Record a freshly spawned first process as running. On failure the
3199    /// module never reaches the roster, so its configured mark is taken back.
3200    fn mark_first_process_running(
3201        &self,
3202        spec: &ModuleSpec,
3203        runtime: &SupervisorRuntimeConfig,
3204        snapshot: &SharedSnapshot,
3205        child: &mut SupervisedChild,
3206    ) -> Result<(), SuperviseError> {
3207        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3208            self.abandon_unrostered(&spec.module_id);
3209            return Err(err);
3210        }
3211        self.process_liveness
3212            .track(spec.module_id.clone(), Arc::clone(snapshot));
3213        Ok(())
3214    }
3215
3216    /// Start supervising a module declared in daemon configuration.
3217    ///
3218    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3219    /// failures in the supervisor handle so operator-facing `supervisor.list`
3220    /// reflects every configured module while daemon startup continues.
3221    pub fn supervise_configured(
3222        &self,
3223        spec: ModuleSpec,
3224        enabled: bool,
3225    ) -> Result<SupervisedModule, SuperviseError> {
3226        self.supervise_configured_requested(
3227            spec,
3228            enabled,
3229            self.health.clone(),
3230            None,
3231            self.restart_policy,
3232            DEFAULT_REGISTER_TIMEOUT,
3233            None,
3234        )
3235    }
3236
3237    /// Supervise a configured module with its own health, drain, and crash
3238    /// budget. The restart policy is per-module because the config file is:
3239    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3240    /// module that is expensive to restart should not be forced onto the same
3241    /// budget as one that is cheap.
3242    pub fn supervise_configured_with_health(
3243        &self,
3244        spec: ModuleSpec,
3245        enabled: bool,
3246        health: HealthConfig,
3247        drain_timeout_ms: Option<u64>,
3248        restart_policy: RestartPolicy,
3249    ) -> Result<SupervisedModule, SuperviseError> {
3250        self.supervise_configured_requested(
3251            spec,
3252            enabled,
3253            health,
3254            drain_timeout_ms,
3255            restart_policy,
3256            DEFAULT_REGISTER_TIMEOUT,
3257            None,
3258        )
3259    }
3260
3261    #[allow(clippy::too_many_arguments)]
3262    pub(crate) fn supervise_configured_requested(
3263        &self,
3264        spec: ModuleSpec,
3265        enabled: bool,
3266        health: HealthConfig,
3267        drain_timeout_ms: Option<u64>,
3268        restart_policy: RestartPolicy,
3269        register_timeout: Duration,
3270        requester: Option<Principal>,
3271    ) -> Result<SupervisedModule, SuperviseError> {
3272        let registered = self.register_configured_with_health(
3273            spec,
3274            enabled,
3275            health,
3276            drain_timeout_ms,
3277            restart_policy,
3278            register_timeout,
3279        )?;
3280        let daemon_initiated = requester.is_none();
3281        self.start_registered_configured(registered, requester, daemon_initiated)
3282    }
3283
3284    /// Record a configured module as starting or disabled, without launching a
3285    /// process or running its monitor. Bootstrap registers the entire roster
3286    /// before publishing discovery, then activates it once children can read
3287    /// the connection file passed in their `--subc` argument.
3288    pub(crate) fn register_configured_with_health(
3289        &self,
3290        spec: ModuleSpec,
3291        enabled: bool,
3292        health: HealthConfig,
3293        drain_timeout_ms: Option<u64>,
3294        restart_policy: RestartPolicy,
3295        register_timeout: Duration,
3296    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3297        validate_spec(&spec)?;
3298        self.establish_identity(&spec);
3299
3300        let mut runtime = self.runtime_config();
3301        runtime.health = health;
3302        runtime.restart_policy = restart_policy;
3303        runtime.register_timeout = register_timeout;
3304        if let Some(ms) = drain_timeout_ms {
3305            runtime.drain_timeout = Duration::from_millis(ms);
3306            *runtime
3307                .effective_drain_timeout
3308                .lock()
3309                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3310        }
3311        let snapshot = if enabled {
3312            SupervisorSnapshot::starting()
3313        } else {
3314            SupervisorSnapshot::disabled()
3315        };
3316        let snapshot = {
3317            let mut snapshot = snapshot;
3318            if enabled {
3319                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3320                {
3321                    snapshot.waiting_for_previous_pid = Some(pid);
3322                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3323                }
3324            }
3325            snapshot
3326        };
3327        let (module, commands) =
3328            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3329        Ok(RegisteredConfiguredModule {
3330            spec,
3331            runtime,
3332            module,
3333            commands,
3334        })
3335    }
3336
3337    /// Activate a previously registered module. Queued control requests are
3338    /// served by the monitor after the initial configured spawn has settled.
3339    pub(crate) fn start_registered_configured(
3340        &self,
3341        registered: RegisteredConfiguredModule,
3342        requester: Option<Principal>,
3343        daemon_initiated: bool,
3344    ) -> Result<SupervisedModule, SuperviseError> {
3345        let RegisteredConfiguredModule {
3346            spec,
3347            mut runtime,
3348            module,
3349            commands,
3350        } = registered;
3351        runtime.start_requester = requester;
3352        runtime.recovery_is_daemon = daemon_initiated;
3353        let snapshot = &module.inner.snapshot;
3354        let should_spawn = {
3355            let state = lock_snapshot(snapshot)?;
3356            state.enabled && state.waiting_for_previous_pid.is_none()
3357        };
3358        let child = if should_spawn {
3359            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3360            let registration_watch = watch_registration(&self.registry, &spec);
3361            let spawned = spawn_child(
3362                &spec,
3363                runtime.connection_file_path.as_deref(),
3364                self.supervisor_handle.as_ref(),
3365                &runtime.stderr_ring,
3366                runtime.capture_logs_dir.as_deref(),
3367                &runtime.child_roster,
3368                #[cfg(target_os = "linux")]
3369                runtime.cgroup_placement.as_ref(),
3370            );
3371            #[cfg(test)]
3372            self.test_after_first_spawn.run(&spec.module_id);
3373            match spawned {
3374                Ok(mut child) => {
3375                    arm_registration_deadline(
3376                        &mut child,
3377                        runtime.register_timeout,
3378                        registration_watch,
3379                    );
3380                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3381                    if let Some(spawn) = activity_spawn {
3382                        spawn.complete(snapshot);
3383                    }
3384                    Some(child)
3385                }
3386                Err(err) => {
3387                    if runtime.health.critical {
3388                        error!(
3389                            module_id = %spec.module_id,
3390                            program = %spec.program.display(),
3391                            error = %err,
3392                            "critical configured module failed to spawn; marking failed and alerting"
3393                        );
3394                    } else {
3395                        error!(
3396                            module_id = %spec.module_id,
3397                            program = %spec.program.display(),
3398                            error = %err,
3399                            "configured module failed to spawn; marking failed and continuing"
3400                        );
3401                    }
3402                    fail_snapshot(
3403                        snapshot,
3404                        Some(&spec.module_id),
3405                        None,
3406                        &runtime.failure_context(daemon_initiated),
3407                    );
3408                    None
3409                }
3410            }
3411        } else {
3412            None
3413        };
3414        self.start_monitor(spec, runtime, &module, child, commands);
3415        Ok(module)
3416    }
3417
3418    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3419        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3420        SupervisorRuntimeConfig {
3421            registry: self.registry.clone(),
3422            recovery_is_daemon: true,
3423            start_requester: None,
3424            pending_activity_restart: false,
3425            provenance_probe: self.provenance_probe.clone(),
3426            scheduled_respawn: Arc::default(),
3427            deferred_reload_reply: Arc::default(),
3428            restart_policy: self.restart_policy,
3429            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3430            drain_timeout: self.drain_timeout,
3431            // Shared with this module's roster copy: daemon shutdown waits on
3432            // each child for the module's own drain budget, as resolved now.
3433            child_roster: self
3434                .child_roster
3435                .for_module(Arc::clone(&effective_drain_timeout)),
3436            effective_drain_timeout,
3437            default_drain_timeout: self.drain_timeout,
3438            health: self.health.clone(),
3439            connection_file_path: self.connection_file_path.clone(),
3440            capture_logs_dir: self.capture_logs_dir.clone(),
3441            forwarding: self.forwarding.clone(),
3442            supervisor_handle: self.supervisor_handle.clone(),
3443            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3444            terminal_ring: Arc::new(Mutex::new(
3445                TerminalRing::new(
3446                    TerminalRingConfig::default(),
3447                    self.daemon_start_clock.started_at_ms(),
3448                )
3449                .with_start_clock(self.daemon_start_clock)
3450                .with_journal(self.terminal_journal.clone())
3451                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3452            )),
3453            spawn_events: self.spawn_events.clone(),
3454            #[cfg(target_os = "linux")]
3455            cgroup_placement: self.cgroup_placement.clone(),
3456            #[cfg(test)]
3457            test_seed_stale_facts_before_enable_spawn: false,
3458            #[cfg(test)]
3459            test_reload_exit_record_gate: None,
3460        }
3461    }
3462
3463    fn supervised_module(
3464        &self,
3465        spec: ModuleSpec,
3466        runtime: SupervisorRuntimeConfig,
3467        snapshot: SharedSnapshot,
3468        child: Option<SupervisedChild>,
3469    ) -> SupervisedModule {
3470        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3471        self.start_monitor(spec, runtime, &module, child, commands);
3472        module
3473    }
3474
3475    fn register_module(
3476        &self,
3477        spec: &ModuleSpec,
3478        runtime: &SupervisorRuntimeConfig,
3479        snapshot: SharedSnapshot,
3480    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3481        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3482            spec: spec.clone(),
3483            health: runtime.health.clone(),
3484        }));
3485        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3486        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3487        // The module's OWN policy, which may be its per-module config rather than
3488        // the supervisor-wide one; status must report the budget the supervise
3489        // loop actually enforces.
3490        let restart_policy = runtime.restart_policy;
3491        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3492        let (tx, rx) = mpsc::channel(4);
3493        let module_id = spec.module_id.clone();
3494        let module = SupervisedModule {
3495            inner: Arc::new(SupervisedModuleInner {
3496                module_id: module_id.clone(),
3497                registry: Arc::clone(&self.registry),
3498                snapshot,
3499                configuration,
3500                stderr_ring,
3501                terminal_ring,
3502                commands: tx,
3503                monitor: Mutex::new(None),
3504                restart_policy,
3505                effective_drain_timeout,
3506                provenance_probe: self.provenance_probe.clone(),
3507            }),
3508        };
3509        // Identity gates and the configured mark must precede the roster entry,
3510        // so a client never sees a module without its admission policy.
3511        if let Some(supervisor_handle) = &self.supervisor_handle {
3512            supervisor_handle.insert(module.clone());
3513        }
3514        (module, rx)
3515    }
3516
3517    fn start_monitor(
3518        &self,
3519        spec: ModuleSpec,
3520        runtime: SupervisorRuntimeConfig,
3521        module: &SupervisedModule,
3522        child: Option<SupervisedChild>,
3523        commands: mpsc::Receiver<SupervisorCommand>,
3524    ) {
3525        let monitor = tokio::spawn(supervise_loop(
3526            spec,
3527            runtime,
3528            Arc::clone(&self.registry),
3529            Arc::clone(&self.process_liveness),
3530            Arc::clone(&module.inner.snapshot),
3531            child,
3532            commands,
3533        ));
3534        *module
3535            .inner
3536            .monitor
3537            .lock()
3538            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3539    }
3540}
3541
3542impl Default for Supervisor {
3543    fn default() -> Self {
3544        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3545    }
3546}
3547
3548/// Handle to one supervised child process.
3549#[derive(Clone)]
3550pub struct SupervisedModule {
3551    inner: Arc<SupervisedModuleInner>,
3552}
3553
3554struct SupervisedModuleInner {
3555    module_id: String,
3556    registry: Arc<Registry>,
3557    snapshot: SharedSnapshot,
3558    configuration: Arc<Mutex<SupervisedConfiguration>>,
3559    stderr_ring: Arc<Mutex<StderrRing>>,
3560    terminal_ring: Arc<Mutex<TerminalRing>>,
3561    commands: mpsc::Sender<SupervisorCommand>,
3562    monitor: Mutex<Option<JoinHandle<()>>>,
3563    /// Copied from the supervisor's runtime config at spawn so `status()` can
3564    /// report the restart budget without reaching back into the supervisor. The
3565    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3566    restart_policy: RestartPolicy,
3567    effective_drain_timeout: Arc<Mutex<Duration>>,
3568    provenance_probe: ExecutableIdentityProbe,
3569}
3570
3571impl fmt::Debug for SupervisedModule {
3572    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3573        f.debug_struct("SupervisedModule")
3574            .field("module_id", &self.inner.module_id)
3575            .field("status", &self.status())
3576            .finish_non_exhaustive()
3577    }
3578}
3579
3580impl SupervisedModule {
3581    pub fn module_id(&self) -> &str {
3582        &self.inner.module_id
3583    }
3584
3585    /// Test-only: put one probe miss on the streak, the way
3586    /// `handle_health_probe_failure` does, so tests can assert what a later
3587    /// event does to the streak without driving the whole probe loop.
3588    #[cfg(test)]
3589    pub(crate) fn record_health_probe_failure_for_test(
3590        &self,
3591        detail: &str,
3592    ) -> Result<(), SuperviseError> {
3593        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3594            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3595            state.health.detail = Some(detail.to_string());
3596        })
3597    }
3598
3599    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3600        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3601    }
3602
3603    pub(crate) fn registration_end_reason(
3604        &self,
3605    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3606        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3607        Ok(match snapshot.state {
3608            ModuleState::Draining if snapshot.draining_to_replace => {
3609                Some(RegistrationEndReason::SupervisorRestart)
3610            }
3611            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3612            ModuleState::Restarting if snapshot.draining_to_replace => {
3613                Some(RegistrationEndReason::SupervisorRestart)
3614            }
3615            _ => None,
3616        })
3617    }
3618
3619    /// The module's retained stderr, newest lines last.
3620    ///
3621    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3622    /// module, `supervisor.list` renders every module, and putting it in the
3623    /// shared snapshot would make each status read carry a payload almost nobody
3624    /// asked for. Callers that want the text ask for it.
3625    pub fn stderr_tail(
3626        &self,
3627        max_lines: Option<usize>,
3628        max_bytes: Option<usize>,
3629    ) -> StderrTailSnapshot {
3630        self.inner
3631            .stderr_ring
3632            .lock()
3633            .unwrap_or_else(|poisoned| poisoned.into_inner())
3634            .snapshot(max_lines, max_bytes)
3635    }
3636
3637    /// The module's bounded terminal history, oldest retained exit first.
3638    ///
3639    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3640    /// daemon whose in-memory history was necessarily reset.
3641    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3642        self.inner
3643            .terminal_ring
3644            .lock()
3645            .unwrap_or_else(|poisoned| poisoned.into_inner())
3646            .snapshot()
3647    }
3648
3649    /// Retained observations from the current ring and all journal generations.
3650    ///
3651    /// Blocking: this reads the journal files. Async callers use
3652    /// [`Self::read_durable_terminal_history`].
3653    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3654        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3655    }
3656
3657    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3658    /// read (up to every retained generation) never occupies a runtime worker.
3659    /// Fails only if the blocking task could not finish (runtime shutdown or a
3660    /// panic in the read).
3661    pub(crate) async fn read_durable_terminal_history(
3662        &self,
3663    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3664        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3665        let module_id = self.inner.module_id.clone();
3666        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3667            .await
3668    }
3669
3670    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3671        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3672            .map(|(status, _)| status)
3673    }
3674
3675    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3676        let state = lock_snapshot(&self.inner.snapshot)?;
3677        Ok(state
3678            .failed_retry
3679            .pending
3680            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3681            .map(|retry| {
3682                unix_ms_now().saturating_add(
3683                    u64::try_from(
3684                        retry
3685                            .deadline
3686                            .saturating_duration_since(Instant::now())
3687                            .as_millis(),
3688                    )
3689                    .unwrap_or(u64::MAX),
3690                )
3691            }))
3692    }
3693
3694    pub(crate) fn record_deliberate_severance(
3695        &self,
3696        identity: ProcessIdentity,
3697    ) -> Result<bool, SuperviseError> {
3698        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3699        if snapshot.pid != Some(identity.pid)
3700            || snapshot.process_start_time != Some(identity.start_time)
3701        {
3702            return Ok(false);
3703        }
3704        snapshot.deliberate_severance = Some(identity);
3705        Ok(true)
3706    }
3707
3708    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3709    ///
3710    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3711    /// does not produce reader-observability logs.
3712    pub(crate) fn status_for_control(
3713        &self,
3714        caller: &'static str,
3715    ) -> Result<ModuleStatus, SuperviseError> {
3716        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3717            .map(|(status, _)| status)
3718    }
3719
3720    fn status_with_snapshot_lock(
3721        &self,
3722        snapshot: &SharedSnapshot,
3723        caller: Option<&'static str>,
3724    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3725        let mut guard = match caller {
3726            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3727            None => lock_snapshot(snapshot)?,
3728        };
3729        // Read the budget through the pruning path so a reader sees the same
3730        // in-window count the restart decision would use, not a stale total.
3731        let restart_count =
3732            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3733        let snapshot = guard.clone();
3734        drop(guard);
3735        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3736            SuperviseError::StatePoisoned {
3737                module_id: Some(self.inner.module_id.clone()),
3738            }
3739        })?;
3740        let registration_active = self
3741            .inner
3742            .registry
3743            .get_module(&self.inner.module_id)
3744            .map_err(SuperviseError::Registry)?
3745            .is_some();
3746        let protocol = snapshot
3747            .spawned_protocol
3748            .unwrap_or(self.declared_protocol()?);
3749        let running_process =
3750            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3751        // Registration is the difference between the two protocols and the only
3752        // one: a subc module that has not registered cannot serve a request even
3753        // though its process is up, and a `none` module never registers at all,
3754        // so requiring it there would pin `live` to false for the whole life of
3755        // a perfectly healthy process.
3756        let live = match protocol {
3757            ModuleProtocol::Subc => running_process && registration_active,
3758            ModuleProtocol::None => running_process,
3759        };
3760
3761        Ok((
3762            ModuleStatus {
3763                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3764                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3765                module_id: self.inner.module_id.clone(),
3766                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3767                state: snapshot.state,
3768                enabled: snapshot.enabled,
3769                process_alive: snapshot.process_alive,
3770                registration_active,
3771                protocol,
3772                live,
3773                restart_count,
3774                lifetime_restarts: snapshot.lifetime_restarts,
3775                spawn_generation: snapshot.spawn_generation,
3776                max_restarts: self.inner.restart_policy.max_restarts,
3777                restart_window: self.inner.restart_policy.window,
3778                drain_timeout,
3779                restart_backoff: self.inner.restart_policy.backoff,
3780                restart_max_backoff: self.inner.restart_policy.max_backoff,
3781                pid: snapshot.reported_pid(),
3782                spawned_at_ms: snapshot.spawned_at_ms,
3783                spawned_from: snapshot.spawned_from,
3784                launch_spec_digest: snapshot.launch_spec_digest,
3785                process_start_time: snapshot.process_start_time,
3786                last_exit: snapshot.last_exit,
3787                health: snapshot.health,
3788            },
3789            snapshot.spawned_file_identity.clone(),
3790        ))
3791    }
3792
3793    #[cfg(test)]
3794    pub(crate) fn hold_snapshot_for_test(
3795        &self,
3796        acquired: std::sync::mpsc::Sender<()>,
3797        hold: Duration,
3798    ) -> std::thread::JoinHandle<()> {
3799        let snapshot = Arc::clone(&self.inner.snapshot);
3800        std::thread::spawn(move || {
3801            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3802            acquired
3803                .send(())
3804                .expect("test receiver waits for snapshot lock");
3805            std::thread::sleep(hold);
3806        })
3807    }
3808
3809    /// The status and the running-image check for `supervisor.provenance`,
3810    /// taken from one status read. The exec acknowledgement can land between
3811    /// two separate reads, and the reply would then pair "no pid yet" with an
3812    /// image observed after the module started, which describes no single
3813    /// moment.
3814    pub(crate) async fn status_and_running_image_agreement(
3815        &self,
3816    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3817        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3818        let image = self
3819            .inner
3820            .provenance_probe
3821            .observe(
3822                status.pid,
3823                status.spawned_from.as_deref(),
3824                identity,
3825                status.process_start_time,
3826            )
3827            .await;
3828        Ok((status, image))
3829    }
3830
3831    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3832        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3833            Ok(snapshot) => snapshot.clone(),
3834            Err(_) => {
3835                return subc_control::RunningImageAgreement::Unavailable {
3836                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3837                };
3838            }
3839        };
3840        self.inner
3841            .provenance_probe
3842            .observe(
3843                snapshot.reported_pid(),
3844                snapshot.spawned_from.as_deref(),
3845                snapshot.spawned_file_identity.clone(),
3846                snapshot.process_start_time,
3847            )
3848            .await
3849    }
3850
3851    /// Memory and CPU time of the module's current process, read now. Only the
3852    /// process the supervisor spawned is read, not processes it has started.
3853    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3854        #[cfg(windows)]
3855        {
3856            let image = lock_snapshot(&self.inner.snapshot)
3857                .ok()
3858                .and_then(|snapshot| {
3859                    snapshot
3860                        .spawned_file_identity
3861                        .as_ref()
3862                        .and_then(|identity| identity.windows_image.clone())
3863                });
3864            match image {
3865                Some(image) => {
3866                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3867                }
3868                None => subc_control::ChildResourceUsage::Unavailable {
3869                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3870                },
3871            }
3872        }
3873        #[cfg(not(windows))]
3874        {
3875            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3876                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3877                Err(_) => {
3878                    return subc_control::ChildResourceUsage::Unavailable {
3879                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3880                    }
3881                }
3882            };
3883            crate::child_resources::read(pid, start_time)
3884        }
3885    }
3886
3887    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3888        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3889        Ok(match snapshot.state {
3890            ModuleState::Restarting => true,
3891            ModuleState::Failed | ModuleState::Disabled => false,
3892            _ if snapshot.enabled && snapshot.respawn_pending => true,
3893            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3894        })
3895    }
3896
3897    #[cfg(test)]
3898    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3899        self.is_warming_with_snapshot_lock(None)
3900    }
3901
3902    pub(crate) fn is_warming_for_control(
3903        &self,
3904        caller: &'static str,
3905    ) -> Result<bool, SuperviseError> {
3906        self.is_warming_with_snapshot_lock(Some(caller))
3907    }
3908
3909    fn is_warming_with_snapshot_lock(
3910        &self,
3911        caller: Option<&'static str>,
3912    ) -> Result<bool, SuperviseError> {
3913        let snapshot = match caller {
3914            Some(caller) => {
3915                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3916            }
3917            None => lock_snapshot(&self.inner.snapshot)?,
3918        }
3919        .clone();
3920        Ok(matches!(
3921            snapshot.state,
3922            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3923        ))
3924    }
3925
3926    /// Drain the module and stop monitoring it.
3927    pub async fn drain(&self) -> Result<(), SuperviseError> {
3928        self.stop().await
3929    }
3930
3931    pub(crate) async fn retire_requested(
3932        &self,
3933        requester: Option<Principal>,
3934    ) -> Result<(), SuperviseError> {
3935        match self.state()? {
3936            ModuleState::Stopped => return Ok(()),
3937            ModuleState::Failed
3938                if lock_snapshot(&self.inner.snapshot)?
3939                    .failed_retry
3940                    .pending
3941                    .is_none() =>
3942            {
3943                return Ok(())
3944            }
3945            ModuleState::Failed
3946            | ModuleState::Starting
3947            | ModuleState::Running
3948            | ModuleState::Unresponsive
3949            | ModuleState::Restarting
3950            | ModuleState::Draining
3951            | ModuleState::Disabled => {}
3952        }
3953
3954        let (reply_tx, reply_rx) = oneshot::channel();
3955        self.inner
3956            .commands
3957            .send(SupervisorCommand::Retire {
3958                requester,
3959                reply: reply_tx,
3960            })
3961            .await
3962            .map_err(|_| SuperviseError::CommandClosed {
3963                module_id: self.inner.module_id.clone(),
3964            })?;
3965        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3966            module_id: self.inner.module_id.clone(),
3967        })?
3968    }
3969
3970    pub async fn stop(&self) -> Result<(), SuperviseError> {
3971        self.stop_requested(None).await
3972    }
3973
3974    pub(crate) async fn stop_requested(
3975        &self,
3976        requester: Option<Principal>,
3977    ) -> Result<(), SuperviseError> {
3978        match self.state()? {
3979            ModuleState::Stopped => return Ok(()),
3980            ModuleState::Failed
3981                if lock_snapshot(&self.inner.snapshot)?
3982                    .failed_retry
3983                    .pending
3984                    .is_none() =>
3985            {
3986                return Ok(())
3987            }
3988            ModuleState::Failed
3989            | ModuleState::Starting
3990            | ModuleState::Running
3991            | ModuleState::Unresponsive
3992            | ModuleState::Restarting
3993            | ModuleState::Draining
3994            | ModuleState::Disabled => {}
3995        }
3996
3997        let (reply_tx, reply_rx) = oneshot::channel();
3998        self.inner
3999            .commands
4000            .send(SupervisorCommand::Drain {
4001                requester,
4002                reply: reply_tx,
4003            })
4004            .await
4005            .map_err(|_| SuperviseError::CommandClosed {
4006                module_id: self.inner.module_id.clone(),
4007            })?;
4008        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4009            module_id: self.inner.module_id.clone(),
4010        })?
4011    }
4012
4013    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4014        self.restart_requested(drain_timeout_ms, None).await
4015    }
4016
4017    pub(crate) async fn restart_requested(
4018        &self,
4019        drain_timeout_ms: Option<u64>,
4020        requester: Option<Principal>,
4021    ) -> Result<(), SuperviseError> {
4022        let queued_at = Instant::now();
4023        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4024        let (reply_tx, reply_rx) = oneshot::channel();
4025        self.inner
4026            .commands
4027            .send(SupervisorCommand::Restart {
4028                requester,
4029                drain_timeout_ms,
4030                received_at_generation,
4031                queued_at,
4032                reply: reply_tx,
4033            })
4034            .await
4035            .map_err(|_| SuperviseError::CommandClosed {
4036                module_id: self.inner.module_id.clone(),
4037            })?;
4038        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4039            module_id: self.inner.module_id.clone(),
4040        })?
4041    }
4042
4043    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
4044    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4045    /// process then drains in the background of the supervise loop) or has
4046    /// failed, leaving the old process serving.
4047    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4048        self.swap_requested(ready_timeout, None).await
4049    }
4050
4051    pub(crate) async fn swap_requested(
4052        &self,
4053        ready_timeout: Option<Duration>,
4054        requester: Option<Principal>,
4055    ) -> Result<(), SuperviseError> {
4056        let (reply_tx, reply_rx) = oneshot::channel();
4057        self.inner
4058            .commands
4059            .send(SupervisorCommand::Swap {
4060                requester,
4061                ready_timeout,
4062                reply: reply_tx,
4063            })
4064            .await
4065            .map_err(|_| SuperviseError::CommandClosed {
4066                module_id: self.inner.module_id.clone(),
4067            })?;
4068        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4069            module_id: self.inner.module_id.clone(),
4070        })?
4071    }
4072
4073    pub async fn reload(&self) -> Result<(), SuperviseError> {
4074        self.reload_requested(None).await
4075    }
4076
4077    pub(crate) async fn reload_requested(
4078        &self,
4079        requester: Option<Principal>,
4080    ) -> Result<(), SuperviseError> {
4081        let (reply_tx, reply_rx) = oneshot::channel();
4082        self.inner
4083            .commands
4084            .send(SupervisorCommand::Reload {
4085                requester,
4086                reply: reply_tx,
4087            })
4088            .await
4089            .map_err(|_| SuperviseError::CommandClosed {
4090                module_id: self.inner.module_id.clone(),
4091            })?;
4092        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4093            module_id: self.inner.module_id.clone(),
4094        })?
4095    }
4096
4097    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4098        self.set_enabled_requested(enabled, None).await
4099    }
4100
4101    pub(crate) async fn set_enabled_requested(
4102        &self,
4103        enabled: bool,
4104        requester: Option<Principal>,
4105    ) -> Result<bool, SuperviseError> {
4106        let (reply_tx, reply_rx) = oneshot::channel();
4107        self.inner
4108            .commands
4109            .send(SupervisorCommand::SetEnabled {
4110                requester,
4111                enabled,
4112                reply: reply_tx,
4113            })
4114            .await
4115            .map_err(|_| SuperviseError::CommandClosed {
4116                module_id: self.inner.module_id.clone(),
4117            })?;
4118        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4119            module_id: self.inner.module_id.clone(),
4120        })?
4121    }
4122
4123    /// The current process's protocol, or the configured protocol when down.
4124    /// A rescan stores the next launch spec without changing how an existing
4125    /// process registers, serves routes, is probed, or exits.
4126    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4127        let configured = self
4128            .inner
4129            .configuration
4130            .lock()
4131            .map_err(|_| SuperviseError::StatePoisoned {
4132                module_id: Some(self.inner.module_id.clone()),
4133            })?
4134            .spec
4135            .protocol;
4136        let state = lock_snapshot(&self.inner.snapshot)?;
4137        Ok(state.spawned_protocol.unwrap_or(configured))
4138    }
4139
4140    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4141        let configuration =
4142            self.inner
4143                .configuration
4144                .lock()
4145                .map_err(|_| SuperviseError::StatePoisoned {
4146                    module_id: Some(self.inner.module_id.clone()),
4147                })?;
4148        Ok((configuration.spec.clone(), configuration.health.clone()))
4149    }
4150
4151    /// Replace this module's launch spec, keeping its health and drain policy,
4152    /// the way a rescan does for a changed config entry. The running process is
4153    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4154    #[cfg(any(test, feature = "test-support"))]
4155    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4156        let (_, health) = self.configuration()?;
4157        let drain_timeout_ms = u64::try_from(
4158            self.inner
4159                .effective_drain_timeout
4160                .lock()
4161                .unwrap_or_else(|poisoned| poisoned.into_inner())
4162                .as_millis(),
4163        )
4164        .ok();
4165        self.update_configuration(spec, health, drain_timeout_ms)
4166            .await
4167    }
4168
4169    pub(crate) async fn update_configuration(
4170        &self,
4171        spec: ModuleSpec,
4172        health: HealthConfig,
4173        drain_timeout_ms: Option<u64>,
4174    ) -> Result<(), SuperviseError> {
4175        if spec.module_id != self.inner.module_id {
4176            return Err(SuperviseError::InvalidSpec {
4177                reason: "a supervised module's module_id cannot be changed".to_string(),
4178            });
4179        }
4180        validate_spec(&spec)?;
4181        let (reply_tx, reply_rx) = oneshot::channel();
4182        self.inner
4183            .commands
4184            .send(SupervisorCommand::UpdateConfiguration {
4185                spec: spec.clone(),
4186                health: health.clone(),
4187                drain_timeout_ms,
4188                reply: reply_tx,
4189            })
4190            .await
4191            .map_err(|_| SuperviseError::CommandClosed {
4192                module_id: self.inner.module_id.clone(),
4193            })?;
4194        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4195            module_id: self.inner.module_id.clone(),
4196        })?;
4197        let mut configuration =
4198            self.inner
4199                .configuration
4200                .lock()
4201                .map_err(|_| SuperviseError::StatePoisoned {
4202                    module_id: Some(self.inner.module_id.clone()),
4203                })?;
4204        configuration.spec = spec;
4205        configuration.health = health;
4206        Ok(())
4207    }
4208}
4209
4210impl Drop for SupervisedModuleInner {
4211    fn drop(&mut self) {
4212        let Ok(mut monitor) = self.monitor.lock() else {
4213            return;
4214        };
4215        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4216            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4217                state.state = ModuleState::Stopped;
4218                clear_current_process_facts(state);
4219            });
4220            monitor.abort();
4221        }
4222        let _ = monitor.take();
4223    }
4224}
4225
4226#[derive(Debug)]
4227enum SupervisorCommand {
4228    Drain {
4229        requester: Option<Principal>,
4230        reply: oneshot::Sender<Result<(), SuperviseError>>,
4231    },
4232    Retire {
4233        requester: Option<Principal>,
4234        reply: oneshot::Sender<Result<(), SuperviseError>>,
4235    },
4236    Restart {
4237        requester: Option<Principal>,
4238        /// Operator override for this one restart's drain budget, in ms. `None`
4239        /// uses the module's configured/default budget; `Some(0)` cuts
4240        /// immediately (wedge bounce: a stuck request never settles, so
4241        /// waiting only delays recovery).
4242        drain_timeout_ms: Option<u64>,
4243        /// The module's `spawn_generation` when the request was received, before
4244        /// it waited in the command queue. A queued restart whose module has
4245        /// since spawned a newer process is already satisfied (see the handler).
4246        received_at_generation: u64,
4247        /// When the request entered the command queue, so the handler can log
4248        /// how long it waited behind the loop's other work.
4249        queued_at: Instant,
4250        reply: oneshot::Sender<Result<(), SuperviseError>>,
4251    },
4252    Reload {
4253        requester: Option<Principal>,
4254        reply: oneshot::Sender<Result<(), SuperviseError>>,
4255    },
4256    SetEnabled {
4257        requester: Option<Principal>,
4258        enabled: bool,
4259        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4260    },
4261    UpdateConfiguration {
4262        spec: ModuleSpec,
4263        health: HealthConfig,
4264        /// Per-module drain override from the new config; `None` re-resolves to
4265        /// the supervisor-wide default.
4266        drain_timeout_ms: Option<u64>,
4267        reply: oneshot::Sender<()>,
4268    },
4269    Swap {
4270        requester: Option<Principal>,
4271        /// How long the candidate may take to register and declare itself
4272        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4273        ready_timeout: Option<Duration>,
4274        /// Answered at cutover or failure; the incumbent's drain follows.
4275        reply: oneshot::Sender<Result<(), SuperviseError>>,
4276    },
4277}
4278
4279#[derive(Debug)]
4280pub enum SuperviseError {
4281    InvalidSpec {
4282        reason: String,
4283    },
4284    Spawn {
4285        program: PathBuf,
4286        source: io::Error,
4287        cgroup_path: Option<PathBuf>,
4288    },
4289    Cgroup {
4290        module_id: String,
4291        source: io::Error,
4292    },
4293    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4294    /// than spawn a reserved module without its identity binding.
4295    LaunchNonce {
4296        reason: String,
4297    },
4298    Wait {
4299        module_id: String,
4300        source: io::Error,
4301    },
4302    Kill {
4303        module_id: String,
4304        source: io::Error,
4305    },
4306    Forwarding(ForwardingError),
4307    Registry(RegistryError),
4308    ReloadUnavailable {
4309        module_id: String,
4310        reason: String,
4311    },
4312    /// An operator restart/reload was requested for a module that is currently
4313    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4314    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4315    /// by a restart, so these commands are rejected instead of re-enabling it.
4316    Disabled {
4317        module_id: String,
4318    },
4319    ReloadFailed {
4320        module_id: String,
4321        reason: String,
4322    },
4323    RegistrationStillActive {
4324        module_id: String,
4325        waited: Duration,
4326    },
4327    StatePoisoned {
4328        module_id: Option<String>,
4329    },
4330    CommandClosed {
4331        module_id: String,
4332    },
4333    /// A restart or reload arrived while a swap's candidate was warming. The
4334    /// swap owns the module until it cuts over or fails; a stop or disable
4335    /// would have aborted it instead.
4336    SwapInProgress {
4337        module_id: String,
4338    },
4339    /// A swap was refused before anything was spawned.
4340    SwapRefused {
4341        module_id: String,
4342        reason: SwapRefusal,
4343    },
4344    /// A swap spawned a candidate and gave up on it. The candidate has been
4345    /// killed and its slot freed; the incumbent was left serving and was never
4346    /// drained, except in the one `CutoverLost` case described on that arm.
4347    SwapFailed {
4348        module_id: String,
4349        arm: SwapFailureArm,
4350        detail: String,
4351        /// How the candidate exited, when it exited on its own before the
4352        /// supervisor gave up on it.
4353        candidate_exit: Option<ExitReport>,
4354    },
4355}
4356
4357/// Why a swap was refused before a candidate was spawned.
4358#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4359pub enum SwapRefusal {
4360    /// The module's config does not declare `overlap: "safe"`.
4361    OverlapExclusive,
4362    /// The module is not registered, so there is no incumbent to keep serving
4363    /// and nothing a swap would improve on; a plain restart is the tool.
4364    NotRegistered,
4365    /// The module does not speak the subc wire, so a candidate could never
4366    /// register or declare itself ready.
4367    ProtocolNone,
4368    /// The supervisor lacks the forwarding table (to cut routes over) or the
4369    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4370    NotConfigured,
4371    /// A swap is already open for this module.
4372    AlreadySwapping,
4373}
4374
4375impl SwapRefusal {
4376    pub fn as_str(self) -> &'static str {
4377        match self {
4378            Self::OverlapExclusive => "overlap_exclusive",
4379            Self::NotRegistered => "not_registered",
4380            Self::ProtocolNone => "protocol_none",
4381            Self::NotConfigured => "not_configured",
4382            Self::AlreadySwapping => "already_swapping",
4383        }
4384    }
4385}
4386
4387/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4388/// serving and undrained; see `CutoverLost`.
4389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4390pub enum SwapFailureArm {
4391    /// The candidate process could not be started.
4392    SpawnFailed,
4393    /// The candidate did not register within the readiness budget.
4394    NeverRegistered,
4395    /// The candidate registered but did not declare itself ready in time.
4396    NeverReady,
4397    /// The candidate exited before cutover.
4398    CandidateExited,
4399    /// The candidate declared itself ready but failed its health probe.
4400    CandidateUnhealthy,
4401    /// An operator stop, disable or retire arrived while the candidate warmed.
4402    /// The candidate was killed and the operator's command then carried out on
4403    /// the incumbent.
4404    Interrupted,
4405    /// The candidate's connection closed at the moment of cutover. If it
4406    /// closed before forwarding moved, the incumbent is untouched. If it closed
4407    /// between the forwarding and registry halves of cutover, forwarding can no
4408    /// longer route to the incumbent, so the module is restarted plainly.
4409    CutoverLost,
4410}
4411
4412impl SwapFailureArm {
4413    pub fn as_str(self) -> &'static str {
4414        match self {
4415            Self::SpawnFailed => "spawn_failed",
4416            Self::NeverRegistered => "never_registered",
4417            Self::NeverReady => "never_ready",
4418            Self::CandidateExited => "candidate_exited",
4419            Self::CandidateUnhealthy => "candidate_unhealthy",
4420            Self::Interrupted => "interrupted",
4421            Self::CutoverLost => "cutover_lost",
4422        }
4423    }
4424}
4425
4426impl fmt::Display for SuperviseError {
4427    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4428        match self {
4429            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4430            Self::Spawn {
4431                program,
4432                source,
4433                cgroup_path: Some(cgroup_path),
4434            } => write!(
4435                f,
4436                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4437                cgroup_path.display(),
4438                program.display()
4439            ),
4440            Self::Spawn {
4441                program,
4442                source,
4443                cgroup_path: None,
4444            } => write!(
4445                f,
4446                "failed to spawn module '{}': {source}",
4447                program.display()
4448            ),
4449            Self::Cgroup { module_id, source } => {
4450                write!(
4451                    f,
4452                    "failed to prepare cgroup for module '{module_id}': {source}"
4453                )
4454            }
4455            Self::LaunchNonce { reason } => {
4456                write!(
4457                    f,
4458                    "failed to generate reserved-module launch nonce: {reason}"
4459                )
4460            }
4461            Self::Wait { module_id, source } => {
4462                write!(f, "failed to wait for module '{module_id}': {source}")
4463            }
4464            Self::Kill { module_id, source } => {
4465                write!(f, "failed to kill module '{module_id}': {source}")
4466            }
4467            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4468            Self::Registry(err) => write!(f, "registry error: {err}"),
4469            Self::ReloadUnavailable { module_id, reason } => {
4470                write!(f, "reload unavailable for module '{module_id}': {reason}")
4471            }
4472            Self::Disabled { module_id } => {
4473                write!(
4474                    f,
4475                    "module '{module_id}' is disabled; enable it before restart or reload"
4476                )
4477            }
4478            Self::ReloadFailed { module_id, reason } => {
4479                write!(f, "reload failed for module '{module_id}': {reason}")
4480            }
4481            Self::RegistrationStillActive { module_id, waited } => write!(
4482                f,
4483                "module '{module_id}' registration remained active after waiting {waited:?}"
4484            ),
4485            Self::StatePoisoned { module_id } => match module_id {
4486                Some(module_id) => {
4487                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4488                }
4489                None => write!(f, "supervisor state was poisoned"),
4490            },
4491            Self::CommandClosed { module_id } => {
4492                write!(
4493                    f,
4494                    "supervisor command channel for module '{module_id}' is closed"
4495                )
4496            }
4497            Self::SwapInProgress { module_id } => write!(
4498                f,
4499                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4500            ),
4501            Self::SwapRefused { module_id, reason } => match reason {
4502                SwapRefusal::OverlapExclusive => write!(
4503                    f,
4504                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4505                ),
4506                SwapRefusal::NotRegistered => write!(
4507                    f,
4508                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4509                ),
4510                SwapRefusal::ProtocolNone => write!(
4511                    f,
4512                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4513                ),
4514                SwapRefusal::NotConfigured => write!(
4515                    f,
4516                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4517                ),
4518                SwapRefusal::AlreadySwapping => {
4519                    write!(f, "module '{module_id}' is already being swapped")
4520                }
4521            },
4522            Self::SwapFailed {
4523                module_id,
4524                arm,
4525                detail,
4526                ..
4527            } => write!(
4528                f,
4529                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4530                arm.as_str()
4531            ),
4532        }
4533    }
4534}
4535
4536impl Error for SuperviseError {
4537    fn source(&self) -> Option<&(dyn Error + 'static)> {
4538        match self {
4539            Self::Spawn { source, .. }
4540            | Self::Cgroup { source, .. }
4541            | Self::Wait { source, .. }
4542            | Self::Kill { source, .. } => Some(source),
4543            Self::Forwarding(err) => Some(err),
4544            Self::Registry(err) => Some(err),
4545            Self::LaunchNonce { .. }
4546            | Self::InvalidSpec { .. }
4547            | Self::ReloadUnavailable { .. }
4548            | Self::Disabled { .. }
4549            | Self::ReloadFailed { .. }
4550            | Self::RegistrationStillActive { .. }
4551            | Self::StatePoisoned { .. }
4552            | Self::CommandClosed { .. }
4553            | Self::SwapInProgress { .. }
4554            | Self::SwapRefused { .. }
4555            | Self::SwapFailed { .. } => None,
4556        }
4557    }
4558}
4559
4560pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4561    if spec.module_id.trim().is_empty() {
4562        return Err(SuperviseError::InvalidSpec {
4563            reason: "module_id must not be empty".to_string(),
4564        });
4565    }
4566
4567    Ok(())
4568}
4569
4570#[derive(Debug, Default)]
4571struct HealthProbeRuntime {
4572    spawn_generation: Option<u64>,
4573    configured_health: Option<HealthConfig>,
4574    registered_connection: Option<crate::ConnectionId>,
4575    advertised: bool,
4576    next_probe_at: Option<Instant>,
4577    probe_index: u64,
4578}
4579
4580fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4581    lock_snapshot(snapshot)
4582        .ok()
4583        .and_then(|state| state.spawned_protocol)
4584        .unwrap_or(spec.protocol)
4585}
4586
4587impl HealthProbeRuntime {
4588    fn refresh_registration(
4589        &mut self,
4590        spec: &ModuleSpec,
4591        runtime: &SupervisorRuntimeConfig,
4592        registry: &Registry,
4593        snapshot: &SharedSnapshot,
4594    ) {
4595        let generation = lock_snapshot(snapshot)
4596            .ok()
4597            .map(|state| state.spawn_generation);
4598        if self.spawn_generation != generation {
4599            // A replacement cannot reuse a deadline that is already due for its
4600            // predecessor. HTTP children get a full cadence to start; wire
4601            // children still wait for their own health-advertising HELLO.
4602            *self = Self {
4603                spawn_generation: generation,
4604                ..Self::default()
4605            };
4606        }
4607        if self.configured_health.as_ref() != Some(&runtime.health) {
4608            self.configured_health = Some(runtime.health.clone());
4609            self.next_probe_at = None;
4610            self.registered_connection = None;
4611            self.probe_index = 0;
4612        }
4613        // A non-wire process never registers. Only an explicitly configured
4614        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4615        // health failure for that kind of process.
4616        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4617            self.registered_connection = None;
4618            self.advertised = runtime.health.http.is_some();
4619            if !self.advertised {
4620                self.next_probe_at = None;
4621                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622                    let unknown = ModuleHealthStatus::default();
4623                    if state.health != unknown {
4624                        state.health = unknown;
4625                    }
4626                });
4627            } else if self.next_probe_at.is_none() {
4628                self.next_probe_at = Some(
4629                    Instant::now()
4630                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4631                );
4632            }
4633            return;
4634        }
4635
4636        let registration = match registry.get_module(&spec.module_id) {
4637            Ok(registration) => registration,
4638            Err(err) => {
4639                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4640                self.advertised = false;
4641                self.next_probe_at = None;
4642                return;
4643            }
4644        };
4645
4646        let Some(registration) = registration else {
4647            self.registered_connection = None;
4648            self.advertised = false;
4649            self.next_probe_at = None;
4650            return;
4651        };
4652
4653        let advertised = registration
4654            .control_ops
4655            .iter()
4656            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4657        if !advertised {
4658            self.registered_connection = Some(registration.connection_id);
4659            self.advertised = false;
4660            self.next_probe_at = None;
4661            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4662                if state.health.status != SupervisorHealthStatus::Unknown
4663                    || state.health.consecutive_failures != 0
4664                    || state.health.last_probe_ms.is_some()
4665                    || state.health.detail.is_some()
4666                    || state.health.metrics.is_some()
4667                {
4668                    state.health.status = SupervisorHealthStatus::Unknown;
4669                    state.health.consecutive_failures = 0;
4670                    state.health.last_probe_ms = None;
4671                    state.health.detail = None;
4672                    state.health.metrics = None;
4673                }
4674            });
4675            return;
4676        }
4677
4678        let reregistered = self.registered_connection != Some(registration.connection_id);
4679        self.registered_connection = Some(registration.connection_id);
4680        self.advertised = true;
4681        if reregistered || self.next_probe_at.is_none() {
4682            self.probe_index = 0;
4683            self.next_probe_at = Some(
4684                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4685            );
4686            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4687                state.health.status = SupervisorHealthStatus::Unknown;
4688                state.health.consecutive_failures = 0;
4689                state.health.detail = None;
4690                state.health.metrics = None;
4691            });
4692        }
4693    }
4694
4695    fn wake_at(&self) -> Option<Instant> {
4696        if !self.advertised {
4697            return None;
4698        }
4699        self.next_probe_at
4700    }
4701
4702    fn due(&self) -> bool {
4703        self.advertised
4704            && self
4705                .next_probe_at
4706                .is_some_and(|next| Instant::now() >= next)
4707    }
4708
4709    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4710        self.probe_index = self.probe_index.wrapping_add(1);
4711        self.next_probe_at = Some(
4712            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4713        );
4714    }
4715}
4716
4717/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4718///
4719/// This was a struct with a single `message: String`, and every one of the
4720/// fifteen construction sites collapsed into it. Each site knows exactly what it
4721/// saw -- the lane is gone, the module did not answer in time, the module
4722/// answered with the wrong thing -- and `handle_health_probe_failure` then
4723/// treated all of them identically: increment a counter, compare to a threshold,
4724/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4725/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4726///
4727/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4728///
4729/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4730///   answer on it again.
4731/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4732///   AND with a perfectly healthy one that lost a CPU race -- which is what
4733///   happens under machine load, and is how this supervisor killed a healthy
4734///   module three times in one day.
4735/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4736///   Restarting on it is defensible, but it is not the silence case and should
4737///   never be counted as one.
4738/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4739///   anything, so it cannot be evidence about the module at all.
4740///
4741/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4742/// one that fires most often, and while every variant collapsed into one string
4743/// it carried the same weight as the strongest.
4744///
4745/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4746/// DESIGN and a reader stopping at it gets the build backwards: the restart
4747/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4748/// probes still increment the failure streak and drive escalation at the
4749/// threshold (see `is_proof_of_death` below for why that is deliberate and
4750/// what gates the change). Absence of evidence restarts modules today.
4751#[derive(Debug)]
4752enum HealthProbeEvidence {
4753    /// The module's control lane is gone. Proof of death.
4754    LaneDead,
4755    /// No reply within the deadline. Proves nothing about the module's state.
4756    NoAnswer,
4757    /// The module replied, but not with a usable health report. Proves it is alive.
4758    BadAnswer,
4759    /// The daemon could not ask. Says nothing about the module.
4760    Misconfigured,
4761}
4762
4763#[derive(Debug)]
4764struct HealthProbeError {
4765    evidence: HealthProbeEvidence,
4766    message: String,
4767}
4768
4769impl HealthProbeError {
4770    fn lane_dead(message: impl Into<String>) -> Self {
4771        Self::with(HealthProbeEvidence::LaneDead, message)
4772    }
4773
4774    fn no_answer(message: impl Into<String>) -> Self {
4775        Self::with(HealthProbeEvidence::NoAnswer, message)
4776    }
4777
4778    fn bad_answer(message: impl Into<String>) -> Self {
4779        Self::with(HealthProbeEvidence::BadAnswer, message)
4780    }
4781
4782    fn misconfigured(message: impl Into<String>) -> Self {
4783        Self::with(HealthProbeEvidence::Misconfigured, message)
4784    }
4785
4786    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4787        Self {
4788            evidence,
4789            message: message.into(),
4790        }
4791    }
4792
4793    /// Whether this observation is proof the module cannot serve.
4794    ///
4795    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4796    /// variant that fires under CPU starvation, and treating it as proof is the
4797    /// defect this enum exists to make impossible to reintroduce silently.
4798    ///
4799    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4800    /// to restart also needs a bound for the case it excludes -- a genuinely
4801    /// wedged module, alive but never answering -- and that bound must come from
4802    /// the distribution of real late-answer latencies, which nothing measures
4803    /// yet. Landing the classification first makes the later change a one-line
4804    /// decision against evidence that already exists, rather than two unproven
4805    /// changes at once.
4806    #[allow(dead_code)]
4807    fn is_proof_of_death(&self) -> bool {
4808        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4809    }
4810
4811    /// Short stable label for logs and the health snapshot.
4812    ///
4813    /// An operator reading `ck health` currently cannot tell "the module is gone"
4814    /// from "the module did not answer in five seconds", because both render as
4815    /// prose in the same field. These labels are what make the two
4816    /// distinguishable at a glance, and they are what a later restart-policy
4817    /// change will be argued from.
4818    fn label(&self) -> &'static str {
4819        match self.evidence {
4820            HealthProbeEvidence::LaneDead => "lane-dead",
4821            HealthProbeEvidence::NoAnswer => "no-answer",
4822            HealthProbeEvidence::BadAnswer => "bad-answer",
4823            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4824        }
4825    }
4826}
4827
4828impl fmt::Display for HealthProbeError {
4829    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4830        f.write_str(&self.message)
4831    }
4832}
4833
4834async fn run_health_probe_cycle(
4835    spec: &ModuleSpec,
4836    runtime: &SupervisorRuntimeConfig,
4837    registry: &Registry,
4838    process_liveness: &SupervisorProcessLiveness,
4839    snapshot: &SharedSnapshot,
4840    child: &mut Option<SupervisedChild>,
4841) {
4842    let now_ms = unix_ms_now();
4843    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4844        .then_some(runtime.health.http.as_deref())
4845        .flatten();
4846    let result = match http {
4847        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4848        None => probe_module_health(&spec.module_id, runtime, None).await,
4849    };
4850    match result {
4851        Ok(report) => {
4852            handle_health_report(
4853                spec,
4854                runtime,
4855                registry,
4856                process_liveness,
4857                snapshot,
4858                child,
4859                report,
4860                now_ms,
4861            )
4862            .await;
4863        }
4864        Err(err) => {
4865            if http.is_some() {
4866                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4867                    state.health.status = SupervisorHealthStatus::Failing;
4868                });
4869            }
4870            handle_health_probe_failure(
4871                spec,
4872                runtime,
4873                registry,
4874                process_liveness,
4875                snapshot,
4876                child,
4877                err,
4878                now_ms,
4879            )
4880            .await;
4881        }
4882    }
4883}
4884
4885pub(crate) struct HttpProbeTarget<'a> {
4886    address: std::net::SocketAddr,
4887    localhost: bool,
4888    authority: &'a str,
4889    path: String,
4890}
4891
4892/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4893/// or TLS. A URL cannot turn a local health check into an outbound connection.
4894pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4895    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4896        return Err("must not contain whitespace, controls, or a fragment".into());
4897    }
4898    let rest = url
4899        .strip_prefix("http://")
4900        .ok_or("must use plain http://")?;
4901    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4902    let (authority, suffix) = rest.split_at(split);
4903    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4904        ("::1", rest)
4905    } else {
4906        let split = authority.find(':').unwrap_or(authority.len());
4907        authority.split_at(split)
4908    };
4909    let ip: std::net::IpAddr = match host {
4910        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4911        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4912        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4913    };
4914    let port = if port.is_empty() {
4915        80
4916    } else {
4917        port.strip_prefix(':')
4918            .and_then(|p| p.parse::<u16>().ok())
4919            .filter(|p| *p > 0)
4920            .ok_or("must have a valid nonzero TCP port")?
4921    };
4922    let path = if suffix.is_empty() {
4923        "/".into()
4924    } else if suffix.starts_with('?') {
4925        format!("/{suffix}")
4926    } else {
4927        suffix.into()
4928    };
4929    Ok(HttpProbeTarget {
4930        address: std::net::SocketAddr::new(ip, port),
4931        localhost: host == "localhost",
4932        authority,
4933        path,
4934    })
4935}
4936
4937async fn probe_http_health(
4938    url: &str,
4939    deadline: Duration,
4940) -> Result<HealthReport, HealthProbeError> {
4941    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4942    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4943    // Keep partial diagnostics outside the timed future so cancellation does
4944    // not discard a status line or body bytes already received.
4945    let mut response_status = String::new();
4946    let mut body = Vec::new();
4947    let probe = async {
4948        // Resolve localhost ourselves so a hosts-file override cannot turn
4949        // this into an outbound request, while IPv6-only local servers work.
4950        let connection = match tokio::net::TcpStream::connect(target.address).await {
4951            Err(_) if target.localhost => {
4952                tokio::net::TcpStream::connect((
4953                    std::net::Ipv6Addr::LOCALHOST,
4954                    target.address.port(),
4955                ))
4956                .await
4957            }
4958            result => result,
4959        };
4960        let mut stream = connection.map_err(|error| {
4961            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4962        })?;
4963        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4964            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4965        let mut reader = BufReader::new(stream);
4966        let mut budget = 16 * 1024;
4967        let status = http_line(&mut reader, &mut budget).await?;
4968        let mut words = status.split_ascii_whitespace();
4969        let version = words.next();
4970        let code = words
4971            .next()
4972            .filter(|word| word.len() == 3)
4973            .and_then(|word| word.parse::<u16>().ok());
4974        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4975            || !code.is_some_and(|code| (100..600).contains(&code))
4976        {
4977            return Err(HealthProbeError::bad_answer(format!(
4978                "invalid HTTP status: {status}"
4979            )));
4980        }
4981        let code = code.expect("validated status code");
4982        response_status = status.clone();
4983        let mut length = None;
4984        let mut chunked = false;
4985        loop {
4986            let line = http_line(&mut reader, &mut budget).await?;
4987            if line.is_empty() {
4988                break;
4989            }
4990            if let Some((name, value)) = line.split_once(':') {
4991                if name.eq_ignore_ascii_case("content-length") {
4992                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4993                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4994                    })?);
4995                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4996                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4997                }
4998            }
4999        }
5000        if chunked {
5001            while body.len() < 200 {
5002                let line = http_line(&mut reader, &mut budget).await?;
5003                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5004                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5005                if size == 0 {
5006                    break;
5007                }
5008                let count = size.min((200 - body.len()) as u64) as usize;
5009                let start = body.len();
5010                (&mut reader)
5011                    .take(count as u64)
5012                    .read_to_end(&mut body)
5013                    .await
5014                    .map_err(|error| {
5015                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5016                    })?;
5017                if body.len() - start != count {
5018                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5019                }
5020                if size > count as u64 || body.len() == 200 {
5021                    break;
5022                }
5023                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5024                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5025                }
5026            }
5027        } else {
5028            reader
5029                .take(length.unwrap_or(200).min(200))
5030                .read_to_end(&mut body)
5031                .await
5032                .map_err(|error| {
5033                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5034                })?;
5035        }
5036        if (200..300).contains(&code) {
5037            Ok(HealthReport::ok())
5038        } else {
5039            Err(HealthProbeError::bad_answer(
5040                "HTTP health endpoint returned non-2xx",
5041            ))
5042        }
5043    };
5044    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5045        Err(HealthProbeError::no_answer(format!(
5046            "HTTP probe timed out after {deadline:?}"
5047        )))
5048    });
5049    if let Err(error) = &mut result {
5050        if !response_status.is_empty() {
5051            error.message = format!(
5052                "{}; {response_status}: {}",
5053                error.message,
5054                String::from_utf8_lossy(&body)
5055            );
5056        }
5057    }
5058    result
5059}
5060
5061async fn http_line(
5062    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5063    remaining: &mut usize,
5064) -> Result<String, HealthProbeError> {
5065    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5066    let mut line = Vec::new();
5067    (&mut *reader)
5068        .take(*remaining as u64)
5069        .read_until(b'\n', &mut line)
5070        .await
5071        .map_err(|error| {
5072            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5073        })?;
5074    *remaining -= line.len();
5075    if !line.ends_with(b"\r\n") {
5076        return Err(HealthProbeError::bad_answer(
5077            "HTTP headers are incomplete or exceed 16 KiB",
5078        ));
5079    }
5080    line.truncate(line.len() - 2);
5081    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5082}
5083
5084async fn probe_module_health(
5085    module_id: &str,
5086    runtime: &SupervisorRuntimeConfig,
5087    drain_deadline: Option<Instant>,
5088) -> Result<HealthReport, HealthProbeError> {
5089    let Some(forwarding) = runtime.forwarding.as_ref() else {
5090        return Err(HealthProbeError::misconfigured(
5091            "supervisor was not configured with a forwarding table",
5092        ));
5093    };
5094    let probe_started_at = Instant::now();
5095    let mut deadline = probe_started_at + runtime.health.deadline;
5096    if let Some(drain_deadline) = drain_deadline {
5097        deadline = deadline.min(drain_deadline);
5098    }
5099    let pending = if drain_deadline.is_some() {
5100        forwarding.begin_drain_health_probe_rpc_for(
5101            module_id,
5102            MODULE_CONTROL_OP_HEALTH_CHECK,
5103            probe_started_at,
5104            deadline,
5105        )
5106    } else {
5107        forwarding.begin_health_probe_rpc_for(
5108            module_id,
5109            MODULE_CONTROL_OP_HEALTH_CHECK,
5110            probe_started_at,
5111            deadline,
5112        )
5113    }
5114    .map_err(|err| {
5115        // The endpoint is not registered, so there is no live control lane to
5116        // ask. That is the module being absent, not slow.
5117        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5118    })?;
5119    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5120}
5121
5122/// [`probe_module_health`] for one endpoint rather than the id's active one.
5123///
5124/// A swap probes two processes that no by-id lookup reaches: its candidate
5125/// before cutover, and its superseded incumbent (for busy gauges) while the
5126/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5127/// bounds the by-id drain probe.
5128async fn probe_endpoint_health(
5129    endpoint: crate::ModuleEndpointId,
5130    runtime: &SupervisorRuntimeConfig,
5131    deadline_cap: Option<Instant>,
5132) -> Result<HealthReport, HealthProbeError> {
5133    let Some(forwarding) = runtime.forwarding.as_ref() else {
5134        return Err(HealthProbeError::misconfigured(
5135            "supervisor was not configured with a forwarding table",
5136        ));
5137    };
5138    let probe_started_at = Instant::now();
5139    let mut deadline = probe_started_at + runtime.health.deadline;
5140    if let Some(cap) = deadline_cap {
5141        deadline = deadline.min(cap);
5142    }
5143    let pending = forwarding
5144        .begin_endpoint_health_probe_rpc_for(
5145            endpoint,
5146            MODULE_CONTROL_OP_HEALTH_CHECK,
5147            probe_started_at,
5148            deadline,
5149        )
5150        .map_err(|err| {
5151            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5152        })?;
5153    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5154}
5155
5156/// Send a begun health probe and classify its answer.
5157async fn await_health_probe(
5158    forwarding: &ForwardingTable,
5159    pending: PendingModuleControlRpc,
5160    deadline: Instant,
5161    probe_budget: Duration,
5162) -> Result<HealthReport, HealthProbeError> {
5163    let PendingModuleControlRpc {
5164        endpoint,
5165        module_sink,
5166        negotiated_ver,
5167        corr,
5168        receiver,
5169    } = pending;
5170    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5171        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5172    })?;
5173    let frame = Frame::build_with_version(
5174        negotiated_ver,
5175        FrameType::Request,
5176        control_flags(),
5177        0,
5178        0,
5179        corr,
5180        body,
5181    )
5182    .map_err(|err| {
5183        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5184    })?;
5185
5186    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5187    // blocks waiting for capacity when the module's egress queue is full, and an
5188    // unbounded await here freezes the whole supervision actor (it stops polling
5189    // Child::wait and supervisor commands), making the module unrecoverable
5190    // in-band. On timeout the probe fails like any transport failure.
5191    match timeout_at(deadline, module_sink.send(frame)).await {
5192        Ok(Ok(())) => {}
5193        Ok(Err(err)) => {
5194            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5195            // A closed sink means the module's egress channel is gone -- the
5196            // receiving half is dropped when its connection tears down. Proof.
5197            return Err(HealthProbeError::lane_dead(format!(
5198                "failed to send health.check: {err}"
5199            )));
5200        }
5201        Err(_elapsed) => {
5202            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5203            // A full egress queue means the module is not draining its socket, which
5204            // is consistent with a wedged module AND with one whose reader is merely
5205            // starved. Silence, not proof.
5206            return Err(HealthProbeError::no_answer(
5207                "health.check send timed out before enqueue (module egress full)",
5208            ));
5209        }
5210    }
5211
5212    match timeout_at(deadline, receiver).await {
5213        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5214        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5215        // and those prove it is alive even though the probe failed.
5216        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5217            response.health_report().ok_or_else(|| {
5218                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5219            })
5220        }
5221        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5222            format!("health.check rejected: {}", body.message),
5223        )),
5224        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5225            Err(HealthProbeError::lane_dead(message))
5226        }
5227        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5228            Err(HealthProbeError::bad_answer(message))
5229        }
5230        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5231            Err(HealthProbeError::bad_answer(format!(
5232                "expected module-control op '{expected}', got '{actual}'"
5233            )))
5234        }
5235        // A reply that crosses the deadline before this waiter observes it is
5236        // still proof of life. The forwarding path records its end-to-end latency
5237        // before delivering this classification.
5238        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5239            "module answered health.check after its daemon deadline",
5240        )),
5241        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5242            "health.check waiter was canceled before the module responded",
5243        )),
5244        Err(_) => {
5245            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5246            Err(HealthProbeError::no_answer(format!(
5247                "module did not answer health.check within {probe_budget:?}"
5248            )))
5249        }
5250    }
5251}
5252
5253#[allow(clippy::too_many_arguments)]
5254async fn handle_health_report(
5255    spec: &ModuleSpec,
5256    runtime: &SupervisorRuntimeConfig,
5257    registry: &Registry,
5258    process_liveness: &SupervisorProcessLiveness,
5259    snapshot: &SharedSnapshot,
5260    child: &mut Option<SupervisedChild>,
5261    report: HealthReport,
5262    now_ms: u64,
5263) {
5264    let status = supervisor_health_status(report.status);
5265    let detail = report.detail.clone();
5266    let metrics = truncate_health_metrics(report.metrics);
5267    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5268        state.health.status = status;
5269        state.health.last_probe_ms = Some(now_ms);
5270        state.health.detail = detail.clone();
5271        state.health.metrics = metrics.clone();
5272        state.health.consecutive_failures = 0;
5273        state.failed_retry.observe_health(
5274            report.status == HealthStatus::Ok,
5275            runtime.restart_policy.window,
5276        );
5277    });
5278
5279    let action = match report.status {
5280        HealthStatus::Ok => return,
5281        HealthStatus::Degraded => runtime.health.on_degraded,
5282        HealthStatus::Failing => runtime.health.on_failing,
5283    };
5284    apply_l3_health_action(
5285        spec,
5286        runtime,
5287        registry,
5288        process_liveness,
5289        snapshot,
5290        child,
5291        status,
5292        detail.as_deref(),
5293        action,
5294        now_ms,
5295    )
5296    .await;
5297}
5298
5299#[allow(clippy::too_many_arguments)]
5300async fn handle_health_probe_failure(
5301    spec: &ModuleSpec,
5302    runtime: &SupervisorRuntimeConfig,
5303    registry: &Registry,
5304    process_liveness: &SupervisorProcessLiveness,
5305    snapshot: &SharedSnapshot,
5306    child: &mut Option<SupervisedChild>,
5307    err: HealthProbeError,
5308    now_ms: u64,
5309) {
5310    let threshold = runtime.health.failure_threshold.max(1);
5311    let mut failures = 0;
5312    // Carry the evidence class into the operator-visible detail. Without it,
5313    // "module did not answer within 5s" and "the control lane is gone" are two
5314    // prose strings in the same field, and the reader has to know the codebase to
5315    // tell which one is proof of anything.
5316    let detail = format!("[{}] {err}", err.label());
5317    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5318        state
5319            .failed_retry
5320            .observe_health(false, runtime.restart_policy.window);
5321        // A failed wire probe invalidates the last report, even before the
5322        // restart threshold. HTTP probes already mark failures as Failing.
5323        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5324            state.health.status = SupervisorHealthStatus::Unknown;
5325        }
5326        state.health.last_probe_ms = Some(now_ms);
5327        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5328        state.health.detail = Some(detail.clone());
5329        state.health.metrics = None;
5330        failures = state.health.consecutive_failures;
5331    });
5332
5333    if failures < threshold {
5334        warn!(
5335            module_id = %spec.module_id,
5336            consecutive_failures = failures,
5337            threshold,
5338            evidence = err.label(),
5339            detail = %detail,
5340            "health.check probe failed"
5341        );
5342        return;
5343    }
5344
5345    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5346        state.state = ModuleState::Unresponsive;
5347        state.health.status = SupervisorHealthStatus::Unresponsive;
5348    });
5349    // The evidence class is logged at the kill site because this is the line an
5350    // operator reads after an unexplained restart. A streak of `no-answer` under
5351    // machine load is the known false-positive shape; a `lane-dead` is not.
5352    if runtime.health.critical {
5353        error!(
5354            module_id = %spec.module_id,
5355            status = "unresponsive",
5356            evidence = err.label(),
5357            detail = %detail,
5358            "critical module health alert"
5359        );
5360    } else {
5361        warn!(
5362            module_id = %spec.module_id,
5363            status = "unresponsive",
5364            evidence = err.label(),
5365            detail = %detail,
5366            "module health threshold breached"
5367        );
5368    }
5369    if let Err(err) = health_restart_child(
5370        spec,
5371        runtime,
5372        registry,
5373        process_liveness,
5374        snapshot,
5375        child,
5376        SupervisorHealthStatus::Unresponsive,
5377        Some(&detail),
5378        now_ms,
5379    )
5380    .await
5381    {
5382        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5383    }
5384}
5385
5386#[allow(clippy::too_many_arguments)]
5387async fn apply_l3_health_action(
5388    spec: &ModuleSpec,
5389    runtime: &SupervisorRuntimeConfig,
5390    registry: &Registry,
5391    process_liveness: &SupervisorProcessLiveness,
5392    snapshot: &SharedSnapshot,
5393    child: &mut Option<SupervisedChild>,
5394    status: SupervisorHealthStatus,
5395    detail: Option<&str>,
5396    action: HealthAction,
5397    now_ms: u64,
5398) {
5399    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5400    match action {
5401        HealthAction::Report => {
5402            info!(
5403                module_id = %spec.module_id,
5404                status = ?status,
5405                detail,
5406                "module reported non-ok health"
5407            );
5408        }
5409        HealthAction::Alert => {
5410            error!(
5411                module_id = %spec.module_id,
5412                status = ?status,
5413                detail,
5414                "module health alert"
5415            );
5416        }
5417        HealthAction::Restart => {
5418            if let Err(err) = health_restart_child(
5419                spec,
5420                runtime,
5421                registry,
5422                process_liveness,
5423                snapshot,
5424                child,
5425                status,
5426                detail,
5427                now_ms,
5428            )
5429            .await
5430            {
5431                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5432            }
5433        }
5434    }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn health_restart_child(
5439    spec: &ModuleSpec,
5440    runtime: &SupervisorRuntimeConfig,
5441    registry: &Registry,
5442    process_liveness: &SupervisorProcessLiveness,
5443    snapshot: &SharedSnapshot,
5444    child: &mut Option<SupervisedChild>,
5445    status: SupervisorHealthStatus,
5446    detail: Option<&str>,
5447    now_ms: u64,
5448) -> Result<(), SuperviseError> {
5449    let (enabled, schedule) = {
5450        let mut state = lock_snapshot(snapshot)?;
5451        let enabled = state.enabled;
5452        let schedule = if enabled {
5453            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5454        } else {
5455            None
5456        };
5457        (enabled, schedule)
5458    };
5459
5460    if !enabled {
5461        return Err(SuperviseError::Disabled {
5462            module_id: spec.module_id.clone(),
5463        });
5464    }
5465
5466    if schedule.is_none() {
5467        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5468        error!(
5469            module_id = %spec.module_id,
5470            status = ?status,
5471            detail,
5472            max_restarts = runtime.restart_policy.max_restarts,
5473            window_secs = runtime.restart_policy.window.as_secs(),
5474            reason = %runtime.restart_policy.budget_exhausted_detail(),
5475            "health restart budget exhausted; marking module failed"
5476        );
5477        let stop_notice = begin_forwarding_drain_if_configured(
5478            spec,
5479            runtime,
5480            registry,
5481            snapshot,
5482            Some(true),
5483            RouteCloseReason::Disable,
5484        )
5485        .await?;
5486        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5487            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5488        })?;
5489        let result = drain_optional_child(
5490            &spec.module_id,
5491            spec.protocol,
5492            stop_notice,
5493            registry,
5494            runtime.forwarding.as_deref(),
5495            snapshot,
5496            &runtime.terminal_ring,
5497            &runtime.spawn_events,
5498            child,
5499            runtime.drain_timeout,
5500            ModuleState::Failed,
5501            Some(true),
5502            &runtime.failure_context(true),
5503        )
5504        .await;
5505        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5507                state.failed_retry.schedule(&spec.module_id);
5508            }
5509        })?;
5510        result?;
5511        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5512        return Ok(());
5513    }
5514
5515    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5516    let mut restart_count = 0;
5517    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5518        restart_count = state.crash_restarts.len();
5519        state.state = ModuleState::Unresponsive;
5520        state.health.status = status;
5521        state.health.last_action = Some(HealthAction::Restart.to_string());
5522        state.health.last_action_ms = Some(now_ms);
5523    })?;
5524    warn!(
5525        module_id = %spec.module_id,
5526        status = ?status,
5527        detail,
5528        restart_count,
5529        restart_in_window = schedule.restart_in_window,
5530        delay_ms = schedule.delay.as_millis() as u64,
5531        "health-triggered module restart"
5532    );
5533
5534    let stop_notice = begin_forwarding_drain_if_configured(
5535        spec,
5536        runtime,
5537        registry,
5538        snapshot,
5539        Some(true),
5540        RouteCloseReason::Restart,
5541    )
5542    .await?;
5543    drain_optional_child(
5544        &spec.module_id,
5545        spec.protocol,
5546        stop_notice,
5547        registry,
5548        runtime.forwarding.as_deref(),
5549        snapshot,
5550        &runtime.terminal_ring,
5551        &runtime.spawn_events,
5552        child,
5553        runtime.drain_timeout,
5554        ModuleState::Restarting,
5555        Some(true),
5556        &runtime.failure_context(true),
5557    )
5558    .await?;
5559    schedule_respawn(
5560        runtime,
5561        snapshot,
5562        &spec.module_id,
5563        schedule.delay,
5564        RespawnKind::Spawn,
5565        true,
5566    )
5567}
5568
5569fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5570    if let Some(reply) = runtime
5571        .deferred_reload_reply
5572        .lock()
5573        .unwrap_or_else(|p| p.into_inner())
5574        .take()
5575    {
5576        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5577            module_id: module_id.to_string(),
5578            reason: reason.to_string(),
5579        }));
5580    }
5581}
5582
5583fn schedule_respawn(
5584    runtime: &SupervisorRuntimeConfig,
5585    snapshot: &SharedSnapshot,
5586    module_id: &str,
5587    delay: Duration,
5588    kind: RespawnKind,
5589    daemon_initiated: bool,
5590) -> Result<(), SuperviseError> {
5591    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5592    update_snapshot(snapshot, Some(module_id), |state| {
5593        state.respawn_pending = true
5594    })?;
5595    *runtime
5596        .scheduled_respawn
5597        .lock()
5598        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5599        deadline: Instant::now() + delay,
5600        kind,
5601        daemon_initiated,
5602    });
5603    Ok(())
5604}
5605
5606fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5607    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5608        state.health.last_action = Some(action);
5609        state.health.last_action_ms = Some(now_ms);
5610    });
5611}
5612
5613fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5614    match status {
5615        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5616        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5617        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5618    }
5619}
5620
5621/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5622/// returned to every `supervisor.list` and `supervisor.health` caller.
5623///
5624/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5625/// path: that request exists to return a module's complete metrics object, and
5626/// `ck health <module-id>` documents it as the way to see what the cached view
5627/// truncates. The asymmetry is the feature.
5628///
5629/// So a new caller must decide which side it is on rather than assume the cap is
5630/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5631/// the truncation that path exists to avoid.
5632fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5633    let metrics = metrics?;
5634    match serde_json::to_vec(&metrics) {
5635        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5636            "truncated": true,
5637            "original_bytes": encoded.len(),
5638        })),
5639        Ok(_) | Err(_) => Some(metrics),
5640    }
5641}
5642
5643/// Spread health probes so a fleet-wide restart does not converge them.
5644///
5645/// The delay is derived from the module id and probe index rather than a random
5646/// source, so it is deterministic per module: a module keeps its own offset
5647/// across daemon restarts instead of re-rolling into a collision.
5648fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5649    if cadence.is_zero() {
5650        return Duration::ZERO;
5651    }
5652    let cadence_ms = cadence.as_millis() as u64;
5653    // This early return is REDUNDANT, deliberately, and a mutation run will show
5654    // it surviving removal. Recording why here so the next person to notice does
5655    // not have to re-derive it:
5656    //
5657    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5658    //   a zero cadence and builds the Duration from whole milliseconds, so a
5659    //   sub-millisecond cadence cannot come from config.
5660    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5661    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5662    //   -- exactly what this returns.
5663    //
5664    // Kept as a guard against a future widening of the config parser (accepting
5665    // microseconds, say), which would make the sub-millisecond case reachable.
5666    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5667    // divides by zero. Remove this and nothing changes.
5668    if cadence_ms == 0 {
5669        return cadence;
5670    }
5671    // Note that this never returns less than one cadence, including for the FIRST
5672    // probe. So a freshly registered module reports health `unknown` for a full
5673    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5674    // ready to answer.
5675    //
5676    // That is a property of the supervisor's schedule, not of any module: an
5677    // operator watching a restart sees `unknown` and cannot tell it from a module
5678    // that is slow to warm. Measured on two unrelated modules, both flipping to
5679    // `ok` between 22s and 32s after restart.
5680    //
5681    // Left as-is because spreading the first probe is what keeps a fleet-wide
5682    // restart from firing fourteen simultaneous probes into a cold machine. The
5683    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5684    // that thundering herd for a faster first reading.
5685    let jitter_span = (cadence_ms / 10).max(1);
5686    let hash = module_id.as_bytes().iter().fold(
5687        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5688        |acc, byte| {
5689            acc.wrapping_mul(1099511628211)
5690                .wrapping_add(u64::from(*byte))
5691        },
5692    );
5693    cadence + Duration::from_millis(hash % jitter_span)
5694}
5695
5696#[cfg(test)]
5697mod tests {
5698    use super::*;
5699
5700    #[test]
5701    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5702        let handle = SupervisorHandle::new();
5703        let module_id = "readded-tombstone";
5704        handle.record_rescan_removal(module_id);
5705        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5706
5707        handle.apply_identity_configuration(&ModuleSpec {
5708            module_id: module_id.to_string(),
5709            program: PathBuf::from("/test/module"),
5710            args: Vec::new(),
5711            env: Vec::new(),
5712            reserved: false,
5713            reserved_prefixes: Vec::new(),
5714            protocol: ModuleProtocol::Subc,
5715            overlap: Default::default(),
5716        });
5717
5718        assert!(
5719            handle.removal_tombstone_age_ms(module_id).is_none(),
5720            "a re-added module must not retain a stale removal tombstone"
5721        );
5722    }
5723
5724    /// What one module's owner looked like from the control plane at the
5725    /// instant after its first process was spawned.
5726    #[derive(Debug, PartialEq, Eq)]
5727    struct OwnerInSpawnWindow {
5728        module_id: String,
5729        configured: bool,
5730        on_roster: bool,
5731        admission_refusal: Option<&'static str>,
5732    }
5733
5734    /// A supervised module's process can connect, register, sync its scopes
5735    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5736    /// the module on the roster until afterward; configured supervision does
5737    /// so before spawning. In both cases the owner must
5738    /// already read as configured, so a scoped `route.open` against it is
5739    /// refused as retryable `scope_not_synced` and not as terminal
5740    /// `scope_not_live` ("will never sync").
5741    ///
5742    /// The hook runs just after each initial spawn attempt, before returning to
5743    /// the caller, so no race with a real child's registration is needed.
5744    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5745    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5746        use crate::scopes::ScopeTable;
5747        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5748
5749        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5750        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5751            module_id: module_id.to_string(),
5752            program,
5753            args: Vec::new(),
5754            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5755                .into_iter()
5756                .map(|key| (key.to_string(), dir.path().display().to_string()))
5757                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5758                .collect(),
5759            reserved: false,
5760            reserved_prefixes: Vec::new(),
5761            protocol: ModuleProtocol::Subc,
5762            overlap: Default::default(),
5763        };
5764        let live = super::terminal_history_tests::fake_aft_stub_path();
5765        let missing = dir.path().join("definitely-missing-module");
5766
5767        let handle = SupervisorHandle::new();
5768        let mut supervisor =
5769            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5770                .with_handle(handle.clone());
5771        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5772        let hook_handle = handle.clone();
5773        let hook_observed = Arc::clone(&observed);
5774        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5775            // Exactly what the control plane computes for a scoped route.open
5776            // naming this module as the owner of a scope it has not synced.
5777            let configured = hook_handle.is_configured(module_id);
5778            let selector = ScopeSelector::new(
5779                Principal::Reserved {
5780                    module_id: module_id.to_string(),
5781                },
5782                "s".to_string(),
5783                Some(1),
5784            );
5785            let carrier = Principal::Reserved {
5786                module_id: "carrier".to_string(),
5787            };
5788            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5789                .admit(&carrier, module_id, &selector, configured)
5790            {
5791                Ok(_) => None,
5792                Err(refusal) => Some(refusal.code),
5793            };
5794            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5795                module_id: module_id.to_string(),
5796                configured,
5797                on_roster: hook_handle.get(module_id).is_some(),
5798                admission_refusal,
5799            });
5800        })));
5801
5802        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5803        let configured = supervisor
5804            .supervise_configured(stub("configured", live.clone()), true)
5805            .unwrap();
5806        let with_health = supervisor
5807            .supervise_configured_with_health(
5808                stub("with-health", live.clone()),
5809                true,
5810                HealthConfig::default(),
5811                None,
5812                RestartPolicy::default(),
5813            )
5814            .unwrap();
5815        // The failed-spawn path still puts the module on the roster (as
5816        // failed), so it is configured throughout.
5817        let failed = supervisor
5818            .supervise_configured_with_health(
5819                stub("failed-spawn", missing.clone()),
5820                true,
5821                HealthConfig::default(),
5822                None,
5823                RestartPolicy::default(),
5824            )
5825            .unwrap();
5826        // A failed plain `spawn` puts nothing on the roster, so its mark is
5827        // taken back once the spawn has failed.
5828        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5829
5830        let expected = [
5831            "plain",
5832            "configured",
5833            "with-health",
5834            "failed-spawn",
5835            "spawn-error",
5836        ]
5837        .into_iter()
5838        .map(|module_id| OwnerInSpawnWindow {
5839            module_id: module_id.to_string(),
5840            configured: true,
5841            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5842            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5843        })
5844        .collect::<Vec<_>>();
5845        assert_eq!(*observed.lock().unwrap(), expected);
5846
5847        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5848            assert!(
5849                handle.get(module_id).is_some(),
5850                "{module_id} is on the roster"
5851            );
5852            assert!(
5853                handle.is_configured(module_id),
5854                "{module_id} stays configured"
5855            );
5856        }
5857        assert!(handle.get("spawn-error").is_none());
5858        assert!(
5859            !handle.is_configured("spawn-error"),
5860            "a plain spawn that failed must not leave its module marked configured"
5861        );
5862
5863        // Leaving the roster clears the mark with it.
5864        handle.retire("failed-spawn");
5865        assert!(!handle.is_configured("failed-spawn"));
5866
5867        for module in [plain, configured, with_health] {
5868            module.stop().await.unwrap();
5869        }
5870        drop(failed);
5871    }
5872
5873    /// A configured module reads as starting until the supervisor has admitted
5874    /// its first process (exec confirmed, identity durably recorded), even
5875    /// though that process is already running by then and can register over
5876    /// HELLO, which is what lists it in catalog.list. So a client that needs
5877    /// the supervisor state must wait for that state; seeing the module in
5878    /// the catalog does not mean supervisor.list already says running.
5879    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5880    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5881        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5882        let spec = ModuleSpec {
5883            module_id: "spawned-not-admitted".to_string(),
5884            program: super::terminal_history_tests::fake_aft_stub_path(),
5885            args: Vec::new(),
5886            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5887                .into_iter()
5888                .map(|key| (key.to_string(), dir.path().display().to_string()))
5889                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5890                .collect(),
5891            reserved: false,
5892            reserved_prefixes: Vec::new(),
5893            protocol: ModuleProtocol::Subc,
5894            overlap: Default::default(),
5895        };
5896        let handle = SupervisorHandle::new();
5897        let mut supervisor =
5898            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5899                .with_handle(handle.clone());
5900        let observed = Arc::new(Mutex::new(None));
5901        let hook_handle = handle.clone();
5902        let hook_observed = Arc::clone(&observed);
5903        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5904            let state = hook_handle
5905                .get(module_id)
5906                .expect("a configured module is on the roster before it is spawned")
5907                .state()
5908                .unwrap();
5909            *hook_observed.lock().unwrap() = Some(state);
5910        })));
5911
5912        let module = supervisor.supervise_configured(spec, true).unwrap();
5913        assert_eq!(
5914            *observed.lock().unwrap(),
5915            Some(ModuleState::Starting),
5916            "between the spawn and its admission the supervisor still reports starting"
5917        );
5918        assert_eq!(module.state().unwrap(), ModuleState::Running);
5919        module.stop().await.unwrap();
5920    }
5921
5922    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5923        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5924        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5925            snapshot.process_alive = true;
5926            snapshot.pid = Some(41);
5927            snapshot.spawned_at_ms = Some(42);
5928            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5929            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5930            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5931                device: 43,
5932                inode: 44,
5933                #[cfg(windows)]
5934                windows_image: None,
5935            });
5936        })
5937        .unwrap();
5938        snapshot
5939    }
5940
5941    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5942        let snapshot = lock_snapshot(snapshot).unwrap();
5943        assert!(!snapshot.process_alive);
5944        assert_eq!(snapshot.pid, None);
5945        assert_eq!(snapshot.spawned_at_ms, None);
5946        assert_eq!(snapshot.spawned_from, None);
5947        assert_eq!(snapshot.launch_spec_digest, None);
5948        assert_eq!(snapshot.spawned_file_identity, None);
5949    }
5950
5951    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5952    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5953        let supervisor =
5954            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5955        let mut runtime = supervisor.runtime_config();
5956        runtime.test_seed_stale_facts_before_enable_spawn = true;
5957        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5958        let mut child = None;
5959        let spec = ModuleSpec {
5960            module_id: "failed-enable-clears-facts".to_string(),
5961            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5962            args: Vec::new(),
5963            env: Vec::new(),
5964            reserved: false,
5965            reserved_prefixes: Vec::new(),
5966            protocol: ModuleProtocol::Subc,
5967            overlap: Default::default(),
5968        };
5969
5970        let result = set_child_enabled(
5971            &spec,
5972            &runtime,
5973            &supervisor.registry,
5974            &supervisor.process_liveness,
5975            &snapshot,
5976            &mut child,
5977            true,
5978            &runtime.failure_context(false),
5979        )
5980        .await;
5981
5982        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5983        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5984        assert_snapshot_process_facts_cleared(&snapshot);
5985    }
5986
5987    #[tokio::test]
5988    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5989        let supervisor =
5990            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5991        let runtime = supervisor.runtime_config();
5992        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5993            ModuleState::Restarting,
5994            true,
5995        )));
5996        let spec = ModuleSpec {
5997            module_id: "start-stranded-restarting".to_string(),
5998            program: super::terminal_history_tests::fake_aft_stub_path(),
5999            args: Vec::new(),
6000            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6001            reserved: false,
6002            reserved_prefixes: Vec::new(),
6003            protocol: ModuleProtocol::None,
6004            overlap: Default::default(),
6005        };
6006        let mut child = None;
6007        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6008        assert!(!super::set_child_enabled(
6009            &spec,
6010            &runtime,
6011            &Registry::default(),
6012            &supervisor.process_liveness,
6013            &snapshot,
6014            &mut child,
6015            true,
6016            &runtime.failure_context(false),
6017        )
6018        .await
6019        .unwrap());
6020        assert!(child.is_none());
6021        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6022        assert!(super::set_child_enabled(
6023            &spec,
6024            &runtime,
6025            &Registry::default(),
6026            &supervisor.process_liveness,
6027            &snapshot,
6028            &mut child,
6029            true,
6030            &runtime.failure_context(false),
6031        )
6032        .await
6033        .unwrap());
6034        assert_eq!(
6035            lock_snapshot(&snapshot).unwrap().state,
6036            ModuleState::Running
6037        );
6038        let mut child = child.unwrap();
6039        child.start_kill().unwrap();
6040        child.wait().await.unwrap();
6041    }
6042
6043    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6044    async fn failed_reload_spawn_clears_current_process_facts() {
6045        let supervisor =
6046            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6047        let mut runtime = supervisor.runtime_config();
6048        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6049        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6050        let mut child = None;
6051        let spec = ModuleSpec {
6052            module_id: "failed-reload-clears-facts".to_string(),
6053            program: PathBuf::from("/unused/failed-reload-module"),
6054            args: Vec::new(),
6055            env: Vec::new(),
6056            reserved: false,
6057            reserved_prefixes: Vec::new(),
6058            protocol: ModuleProtocol::Subc,
6059            overlap: Default::default(),
6060        };
6061
6062        let result = handle_reload_spawn_failure(
6063            &spec,
6064            &runtime,
6065            &supervisor.process_liveness,
6066            &snapshot,
6067            &mut child,
6068            "forced reload spawn failure".to_string(),
6069        )
6070        .await;
6071
6072        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6073        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6074        assert_snapshot_process_facts_cleared(&snapshot);
6075    }
6076
6077    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6078    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6079        let supervisor =
6080            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6081        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6082        let module = supervisor.supervised_module(
6083            ModuleSpec {
6084                module_id: "drop-clears-facts".to_string(),
6085                program: PathBuf::from("/unused/drop-module"),
6086                args: Vec::new(),
6087                env: Vec::new(),
6088                reserved: false,
6089                reserved_prefixes: Vec::new(),
6090                protocol: ModuleProtocol::Subc,
6091                overlap: Default::default(),
6092            },
6093            supervisor.runtime_config(),
6094            Arc::clone(&snapshot),
6095            None,
6096        );
6097        assert!(!module
6098            .inner
6099            .monitor
6100            .lock()
6101            .unwrap()
6102            .as_ref()
6103            .unwrap()
6104            .is_finished());
6105
6106        drop(module);
6107
6108        assert_eq!(
6109            lock_snapshot(&snapshot).unwrap().state,
6110            ModuleState::Stopped
6111        );
6112        assert_snapshot_process_facts_cleared(&snapshot);
6113    }
6114
6115    #[cfg(unix)]
6116    #[tokio::test]
6117    async fn rescan_preserves_running_protocol_until_respawn() {
6118        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6119        let initial = ModuleSpec {
6120            module_id: "rescan-protocol".into(),
6121            program: PathBuf::from("/bin/sleep"),
6122            args: vec!["60".into()],
6123            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6124                .into_iter()
6125                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6126                .collect(),
6127            reserved: false,
6128            reserved_prefixes: vec![],
6129            protocol: ModuleProtocol::None,
6130            overlap: Default::default(),
6131        };
6132        let supervisor =
6133            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6134        let module = supervisor.spawn(initial.clone()).unwrap();
6135        assert!(module.status().unwrap().live);
6136        let mut next = initial;
6137        next.protocol = ModuleProtocol::Subc;
6138        module
6139            .update_configuration(next.clone(), HealthConfig::default(), None)
6140            .await
6141            .unwrap();
6142        assert!(
6143            module.status().unwrap().live,
6144            "rescan must not require HELLO from the old non-wire process"
6145        );
6146        let runtime = supervisor.runtime_config();
6147        let action = on_child_exit(
6148            &next,
6149            RestartPolicy::default(),
6150            &supervisor.registry,
6151            &module.inner.snapshot,
6152            &runtime.terminal_ring,
6153            &runtime.spawn_events,
6154            &runtime.child_roster,
6155            ExitReport {
6156                kind: ExitKind::Clean,
6157                code: Some(0),
6158                signal: None,
6159                at_ms: unix_ms_now(),
6160            },
6161            true,
6162        )
6163        .await;
6164        assert!(
6165            matches!(action, NextAction::Restart { .. }),
6166            "the old non-wire process's clean exit must restart"
6167        );
6168        module.drain().await.unwrap();
6169    }
6170
6171    #[cfg(unix)]
6172    #[tokio::test]
6173    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6174        use std::os::unix::fs::PermissionsExt;
6175        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6176        let script = dir.join("module.sh");
6177        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6178        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6179        let record_path = dir.join("live-children.json");
6180        let supervisor =
6181            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6182                .with_live_children_record(&record_path);
6183        for (program, args) in [
6184            (PathBuf::from("sleep"), vec!["60".into()]),
6185            (script, vec![]),
6186        ] {
6187            let spec = ModuleSpec {
6188                module_id: "image-identity".into(),
6189                program: program.clone(),
6190                args,
6191                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6192                    .into_iter()
6193                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6194                    .collect(),
6195                reserved: false,
6196                reserved_prefixes: vec![],
6197                protocol: ModuleProtocol::None,
6198                overlap: Default::default(),
6199            };
6200            let module = supervisor.spawn(spec).unwrap();
6201            #[cfg(target_os = "macos")]
6202            {
6203                // SETEXEC confirmation is asynchronous; the orphan record must
6204                // identify the final image, never the intermediate trampoline.
6205                let deadline = Instant::now() + Duration::from_secs(5);
6206                while crate::live_children::read_record(&record_path)
6207                    .unwrap()
6208                    .iter()
6209                    .all(|entry| entry.executable.is_none())
6210                {
6211                    assert!(Instant::now() < deadline, "module image was not confirmed");
6212                    tokio::time::sleep(Duration::from_millis(5)).await;
6213                }
6214            }
6215            let entry = crate::live_children::read_record(&record_path)
6216                .unwrap()
6217                .pop()
6218                .unwrap();
6219            let observed = subc_os::Process::open(entry.pid)
6220                .unwrap()
6221                .unwrap()
6222                .observe()
6223                .unwrap();
6224            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6225            module.drain().await.unwrap();
6226            assert_eq!(
6227                verdict,
6228                crate::live_children::IdentityVerdict::Matches,
6229                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6230            );
6231        }
6232    }
6233
6234    #[cfg(unix)]
6235    fn http_fixture(
6236        dir: &std::path::Path,
6237        url: &str,
6238        threshold: u32,
6239    ) -> crate::daemon_config::ConfiguredModule {
6240        let path = dir.join("subc.jsonc");
6241        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6242            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6243            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6244            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6245            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6246        }}}).to_string()).unwrap();
6247        crate::daemon_config::load(&path)
6248            .unwrap()
6249            .unwrap()
6250            .modules
6251            .pop()
6252            .unwrap()
6253    }
6254
6255    #[cfg(unix)]
6256    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6257        timeout(Duration::from_secs(5), async {
6258            loop {
6259                if module.status().unwrap().health.status == status {
6260                    break;
6261                }
6262                sleep(Duration::from_millis(5)).await;
6263            }
6264        })
6265        .await
6266        .unwrap_or_else(|_| {
6267            panic!(
6268                "expected {status:?}, got {:?}",
6269                module.status().unwrap().health
6270            )
6271        });
6272    }
6273
6274    #[cfg(unix)]
6275    #[tokio::test]
6276    async fn http_health_status_flips_ok_failing_ok() {
6277        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6278        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6279        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6280        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6281        let serving_status = status.clone();
6282        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6283        let server = tokio::spawn(async move {
6284            loop {
6285                let (mut stream, _) = listener.accept().await.unwrap();
6286                let mut request = [0u8; 2048];
6287                let count = stream.read(&mut request).await.unwrap();
6288                assert!(count > 0, "a probe must send an HTTP request");
6289                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6290                let body = if code == 200 {
6291                    "ready"
6292                } else {
6293                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6294                };
6295                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6296                let _ = stream.write_all(response.as_bytes()).await;
6297            }
6298        });
6299        let configured = http_fixture(&dir, &url, 1000);
6300        let module =
6301            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6302                .supervise_configured_with_health(
6303                    configured.module_spec(),
6304                    true,
6305                    configured.health,
6306                    configured.drain_timeout_ms,
6307                    configured.restart,
6308                )
6309                .unwrap();
6310        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6311        status.store(503, std::sync::atomic::Ordering::SeqCst);
6312        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6313        assert!(module
6314            .status()
6315            .unwrap()
6316            .health
6317            .detail
6318            .unwrap()
6319            .contains("scratch failure"));
6320        status.store(200, std::sync::atomic::Ordering::SeqCst);
6321        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6322        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6323        let before = module.status().unwrap();
6324        let (spec, mut health) = module.configuration().unwrap();
6325        health.http = None;
6326        module
6327            .update_configuration(spec.clone(), health.clone(), Some(10))
6328            .await
6329            .unwrap();
6330        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6331        health.http = Some(url);
6332        module
6333            .update_configuration(spec, health, Some(10))
6334            .await
6335            .unwrap();
6336        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6337        assert_eq!(
6338            module.status().unwrap().pid,
6339            before.pid,
6340            "changing a probe must apply live, not restart its process"
6341        );
6342        let (spec, mut health) = module.configuration().unwrap();
6343        health.failure_threshold = 2;
6344        module
6345            .update_configuration(spec, health, Some(10))
6346            .await
6347            .unwrap();
6348        status.store(503, std::sync::atomic::Ordering::SeqCst);
6349        timeout(Duration::from_secs(5), async {
6350            while module.status().unwrap().spawn_generation == before.spawn_generation {
6351                sleep(Duration::from_millis(5)).await;
6352            }
6353        })
6354        .await
6355        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6356        module.drain().await.unwrap();
6357        server.abort();
6358    }
6359
6360    #[cfg(unix)]
6361    #[tokio::test]
6362    async fn http_health_replacement_recovers_after_one_restart() {
6363        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6364        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6365        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6366        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6367        let mut configured = http_fixture(&dir, &url, 2);
6368        configured.restart.max_restarts = 2;
6369        let module =
6370            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6371                .supervise_configured_with_health(
6372                    configured.module_spec(),
6373                    true,
6374                    configured.health,
6375                    configured.drain_timeout_ms,
6376                    configured.restart,
6377                )
6378                .unwrap();
6379        let first_generation = module.status().unwrap().spawn_generation;
6380        let observed = module.clone();
6381        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6382        let server = tokio::spawn(async move {
6383            let mut generation = first_generation;
6384            let mut probes = 0;
6385            loop {
6386                let (mut stream, _) = listener.accept().await.unwrap();
6387                let mut request = [0; 2048];
6388                assert!(stream.read(&mut request).await.unwrap() > 0);
6389                let current = observed.status().unwrap().spawn_generation;
6390                if current != generation {
6391                    generation = current;
6392                    probes = 0;
6393                }
6394                probes += 1;
6395                // The original process never recovers. Each replacement has one
6396                // transient miss, then answers successfully for its remaining life.
6397                let healthy = generation != first_generation && probes > 1;
6398                let code = if healthy { 200 } else { 503 };
6399                let response = format!(
6400                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6401                );
6402                let _ = stream.write_all(response.as_bytes()).await;
6403                if healthy {
6404                    healthy_tx.send(()).await.unwrap();
6405                }
6406            }
6407        });
6408        timeout(Duration::from_secs(5), async {
6409            let mut healthy_probes = 0;
6410            loop {
6411                if module.state().unwrap() == ModuleState::Failed {
6412                    break;
6413                }
6414                while healthy_rx.try_recv().is_ok() {
6415                    healthy_probes += 1;
6416                }
6417                if healthy_probes >= 3
6418                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6419                {
6420                    break;
6421                }
6422                sleep(Duration::from_millis(5)).await;
6423            }
6424        })
6425        .await
6426        .expect("the replacement must settle healthy without exhausting its restart budget");
6427        let status = module.status().unwrap();
6428        module.drain().await.unwrap();
6429        server.abort();
6430        assert_eq!(status.state, ModuleState::Running);
6431        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6432        assert_eq!(status.health.consecutive_failures, 0);
6433        assert_eq!(status.lifetime_restarts, 1);
6434        assert_eq!(status.restart_count, 1);
6435    }
6436
6437    #[cfg(unix)]
6438    #[tokio::test]
6439    async fn http_health_sustained_failure_exhausts_restart_budget() {
6440        use std::collections::BTreeMap;
6441        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6442        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6443        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6444        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6445        let mut configured = http_fixture(&dir, &url, 2);
6446        configured.restart.max_restarts = 2;
6447        let module =
6448            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6449                .supervise_configured_with_health(
6450                    configured.module_spec(),
6451                    true,
6452                    configured.health,
6453                    configured.drain_timeout_ms,
6454                    configured.restart,
6455                )
6456                .unwrap();
6457        let observed = module.clone();
6458        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6459        let served = probes.clone();
6460        let server = tokio::spawn(async move {
6461            loop {
6462                let (mut stream, _) = listener.accept().await.unwrap();
6463                let mut request = [0; 2048];
6464                assert!(stream.read(&mut request).await.unwrap() > 0);
6465                *served
6466                    .lock()
6467                    .unwrap()
6468                    .entry(observed.status().unwrap().spawn_generation)
6469                    .or_default() += 1;
6470                let _ = stream
6471                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6472                    .await;
6473            }
6474        });
6475        timeout(Duration::from_secs(5), async {
6476            while module.state().unwrap() != ModuleState::Failed {
6477                sleep(Duration::from_millis(5)).await;
6478            }
6479        })
6480        .await
6481        .expect("sustained failures must still exhaust the health restart budget");
6482        let status = module.status().unwrap();
6483        server.abort();
6484        assert_eq!(status.restart_count, 2);
6485        assert_eq!(status.lifetime_restarts, 2);
6486        assert!(!status.process_alive);
6487        assert_eq!(
6488            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6489            vec![2, 2, 2]
6490        );
6491    }
6492
6493    #[cfg(unix)]
6494    #[tokio::test]
6495    async fn spawn_clears_the_previous_process_health() {
6496        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6497        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6498        configured.health.cadence = Duration::from_secs(3600);
6499        let module =
6500            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6501                .supervise_configured_with_health(
6502                    configured.module_spec(),
6503                    false,
6504                    configured.health,
6505                    configured.drain_timeout_ms,
6506                    configured.restart,
6507                )
6508                .unwrap();
6509        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6510            state.health.status = SupervisorHealthStatus::Unresponsive;
6511            state.health.consecutive_failures = 3;
6512            state.health.detail = Some("old process refused connections".into());
6513            state.health.metrics = Some(serde_json::json!({"old": true}));
6514            state.health.last_probe_ms = Some(123);
6515            state.health.last_action = Some("restart".into());
6516            state.health.last_action_ms = Some(124);
6517        })
6518        .unwrap();
6519        module.set_enabled(true).await.unwrap();
6520        let health = module.status().unwrap().health;
6521        module.drain().await.unwrap();
6522        assert_eq!(health.consecutive_failures, 0);
6523        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6524        assert_eq!(health.detail, None);
6525        assert_eq!(health.metrics, None);
6526        assert_eq!(health.last_probe_ms, None);
6527        assert_eq!(health.last_action.as_deref(), Some("restart"));
6528        assert_eq!(health.last_action_ms, Some(124));
6529    }
6530
6531    #[cfg(unix)]
6532    #[tokio::test]
6533    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6534        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6535        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6536        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6537        drop(unused);
6538        let configured = http_fixture(&dir, &url, 2);
6539        let module =
6540            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6541                .supervise_configured_with_health(
6542                    configured.module_spec(),
6543                    true,
6544                    configured.health,
6545                    configured.drain_timeout_ms,
6546                    configured.restart,
6547                )
6548                .unwrap();
6549        let before = module.status().unwrap().spawn_generation;
6550        timeout(Duration::from_secs(5), async {
6551            loop {
6552                let status = module.status().unwrap();
6553                if status.spawn_generation > before {
6554                    assert!(status.lifetime_restarts > 0);
6555                    break;
6556                }
6557                sleep(Duration::from_millis(5)).await;
6558            }
6559        })
6560        .await
6561        .expect("sustained HTTP refusal must trigger the health restart policy");
6562        module.drain().await.unwrap();
6563    }
6564
6565    #[cfg(unix)]
6566    #[tokio::test]
6567    async fn http_health_timeout_honours_deadline() {
6568        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6569        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6570        let server = tokio::spawn(async move {
6571            let _held = listener.accept().await.unwrap();
6572            std::future::pending::<()>().await;
6573        });
6574        let error = timeout(
6575            Duration::from_secs(1),
6576            probe_http_health(&url, Duration::from_millis(10)),
6577        )
6578        .await
6579        .expect("the probe must enforce its own deadline")
6580        .unwrap_err();
6581        server.abort();
6582        assert!(error.to_string().contains("timed out"));
6583    }
6584
6585    #[cfg(unix)]
6586    #[tokio::test]
6587    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6588        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6589        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6590        let url = format!(
6591            "http://localhost:{}/healthz",
6592            listener.local_addr().unwrap().port()
6593        );
6594        let server = tokio::spawn(async move {
6595            let (mut stream, _) = listener.accept().await.unwrap();
6596            let mut request = [0u8; 2048];
6597            assert!(stream.read(&mut request).await.unwrap() > 0);
6598            stream
6599                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6600                .await
6601                .unwrap();
6602        });
6603        // The deadline only bounds a hang. A probe that never tried the IPv6
6604        // address would be refused on 127.0.0.1 and fail at once, so a longer
6605        // deadline does not weaken the assertion; one second timed out under a
6606        // loaded parallel test run.
6607        assert_eq!(
6608            probe_http_health(&url, Duration::from_secs(10))
6609                .await
6610                .unwrap()
6611                .status,
6612            HealthStatus::Ok
6613        );
6614        server.await.unwrap();
6615    }
6616
6617    #[cfg(unix)]
6618    #[tokio::test]
6619    async fn http_health_timeout_keeps_partial_status_and_body() {
6620        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6621        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6622        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6623        let server = tokio::spawn(async move {
6624            let (mut stream, _) = listener.accept().await.unwrap();
6625            let mut request = [0u8; 2048];
6626            assert!(stream.read(&mut request).await.unwrap() > 0);
6627            stream
6628                .write_all(
6629                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6630                )
6631                .await
6632                .unwrap();
6633            std::future::pending::<()>().await;
6634        });
6635        let error = probe_http_health(&url, Duration::from_secs(1))
6636            .await
6637            .unwrap_err()
6638            .to_string();
6639        server.abort();
6640        assert!(
6641            error.contains("timed out")
6642                && error.contains("503 Unavailable")
6643                && error.contains("partial diagnostic"),
6644            "{error}"
6645        );
6646    }
6647
6648    #[cfg(unix)]
6649    #[tokio::test]
6650    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6651        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6652        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6653        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6654        let server = tokio::spawn(async move {
6655            let (mut stream, _) = listener.accept().await.unwrap();
6656            let mut request = [0u8; 2048];
6657            assert!(stream.read(&mut request).await.unwrap() > 0);
6658            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6659            let response = format!(
6660                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6661                body.len()
6662            );
6663            stream.write_all(response.as_bytes()).await.unwrap();
6664        });
6665        let error = probe_http_health(&url, Duration::from_secs(1))
6666            .await
6667            .unwrap_err()
6668            .to_string();
6669        server.await.unwrap();
6670        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6671        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6672        assert!(!error.contains("not-in-diagnostic"));
6673    }
6674
6675    #[cfg(unix)]
6676    #[tokio::test]
6677    async fn http_health_real_nats_server_monitoring() {
6678        if std::process::Command::new("nats-server")
6679            .arg("--version")
6680            .env("XDG_DATA_HOME", std::env::temp_dir())
6681            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6682            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6683            .output()
6684            .is_err()
6685        {
6686            eprintln!(
6687                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6688            );
6689            return;
6690        }
6691        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6692        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6693        let port = monitor.local_addr().unwrap().port();
6694        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6695        let client_port = client.local_addr().unwrap().port();
6696        let config = dir.join("server.conf");
6697        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6698        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6699        configured.program = PathBuf::from("nats-server");
6700        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6701        drop(monitor);
6702        drop(client);
6703        let module =
6704            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6705                .supervise_configured_with_health(
6706                    configured.module_spec(),
6707                    true,
6708                    configured.health,
6709                    configured.drain_timeout_ms,
6710                    configured.restart,
6711                )
6712                .unwrap();
6713        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6714        module.drain().await.unwrap();
6715    }
6716
6717    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6718    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6719        let supervisor =
6720            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6721        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6722        let initial = ModuleSpec {
6723            module_id: "rescan-preserves-spawn-facts".to_string(),
6724            program: PathBuf::from("/spawned/module"),
6725            args: Vec::new(),
6726            env: Vec::new(),
6727            reserved: false,
6728            reserved_prefixes: Vec::new(),
6729            protocol: ModuleProtocol::Subc,
6730            overlap: Default::default(),
6731        };
6732        let module = supervisor.supervised_module(
6733            initial.clone(),
6734            supervisor.runtime_config(),
6735            snapshot,
6736            None,
6737        );
6738        let before = module.status().unwrap();
6739        let mut replacement = initial;
6740        replacement.program = PathBuf::from("/rescanned/replacement-module");
6741
6742        module
6743            .update_configuration(replacement, HealthConfig::default(), None)
6744            .await
6745            .unwrap();
6746
6747        let after = module.status().unwrap();
6748        assert_eq!(after.pid, before.pid);
6749        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6750        assert_eq!(after.spawned_from, before.spawned_from);
6751        drop(module);
6752    }
6753}
6754
6755fn unix_ms_now() -> u64 {
6756    SystemTime::now()
6757        .duration_since(UNIX_EPOCH)
6758        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6759        .unwrap_or(0)
6760}
6761
6762async fn supervise_loop(
6763    mut spec: ModuleSpec,
6764    mut runtime: SupervisorRuntimeConfig,
6765    registry: Arc<Registry>,
6766    process_liveness: Arc<SupervisorProcessLiveness>,
6767    snapshot: SharedSnapshot,
6768    mut child: Option<SupervisedChild>,
6769    mut commands: mpsc::Receiver<SupervisorCommand>,
6770) {
6771    let mut health_probe = HealthProbeRuntime::default();
6772    // Registry writes (including embedded callers) notify this module only.
6773    // Subscribe before the first refresh; watch retains changes that arrive
6774    // while commands or probes are running. No polling fallback is needed.
6775    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6776        Ok(changes) => Some(changes),
6777        Err(err) => {
6778            // A poisoned registry cannot accept further writes, so it cannot
6779            // recover via a timer. Keep exit and command handling alive.
6780            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6781            None
6782        }
6783    };
6784    // All restart backoffs run here, including health and operator requests.
6785    // While one is pending the loop serves commands, so disable or drain can
6786    // cancel the replacement without spawning a process just to stop it.
6787    let mut pending_respawn: Option<PendingRespawn> = None;
6788    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6789    // before anything else so a stop that interrupted a swap runs at once.
6790    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6791    let mut previous_instance_recheck = tokio::time::interval_at(
6792        Instant::now() + Duration::from_secs(5),
6793        Duration::from_secs(5),
6794    );
6795    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6796    loop {
6797        #[cfg(test)]
6798        {
6799            lock_snapshot(&snapshot).unwrap().actor_select = None;
6800        }
6801        #[cfg(target_os = "macos")]
6802        if let Some(active) = child.as_mut() {
6803            active.confirm_privacy_exec().await;
6804        }
6805        if let Some(scheduled) = runtime
6806            .scheduled_respawn
6807            .lock()
6808            .unwrap_or_else(|p| p.into_inner())
6809            .take()
6810        {
6811            runtime.recovery_is_daemon = scheduled.daemon_initiated;
6812            pending_respawn = Some(scheduled);
6813        }
6814        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6815            pending_respawn = None;
6816            cancel_deferred_reload(
6817                &runtime,
6818                &spec.module_id,
6819                "respawn cancelled by a supervisor command",
6820            );
6821        }
6822        if child.is_none()
6823            && pending_respawn.is_none()
6824            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6825        {
6826            cancel_deferred_reload(
6827                &runtime,
6828                &spec.module_id,
6829                "respawn cancelled before a replacement was spawned",
6830            );
6831            let mut stranded = false;
6832            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6833                state.respawn_pending = false;
6834                state.coalesced_restart_pending = false;
6835                if matches!(
6836                    state.state,
6837                    ModuleState::Restarting
6838                        | ModuleState::Starting
6839                        | ModuleState::Draining
6840                        | ModuleState::Unresponsive
6841                ) {
6842                    error!(module_id = %spec.module_id, state = ?state.state,
6843                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6844                    stranded = true;
6845                }
6846            });
6847            if stranded {
6848                fail_snapshot(
6849                    &snapshot,
6850                    Some(&spec.module_id),
6851                    None,
6852                    &runtime.failure_context(runtime.recovery_is_daemon),
6853                );
6854            }
6855        }
6856        if let Some(command) = requeued.pop_front() {
6857            if !handle_supervisor_command(
6858                command,
6859                &mut spec,
6860                &mut runtime,
6861                &registry,
6862                &process_liveness,
6863                &snapshot,
6864                &mut child,
6865                &mut commands,
6866                &mut requeued,
6867            )
6868            .await
6869            {
6870                return;
6871            }
6872            if child.is_some() || !respawn_still_pending(&snapshot) {
6873                pending_respawn = None;
6874                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6875                    state.respawn_pending = false
6876                });
6877            }
6878            continue;
6879        }
6880        if child.is_some() {
6881            runtime.recovery_is_daemon = true;
6882            #[cfg(test)]
6883            {
6884                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6885            }
6886            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6887            let active = child.as_mut().expect("child checked above");
6888            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6889                active.registration_deadline = None;
6890            }
6891            let registration_deadline = active.registration_deadline;
6892            let no_checks = runtime.health.http.is_none()
6893                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6894                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6895            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6896                state
6897                    .failed_retry
6898                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
6899            });
6900            let wake_at = match (health_probe.wake_at(), uptime_reset) {
6901                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6902                (probe, None) => probe,
6903            };
6904            let wake_at = match (wake_at, registration_deadline) {
6905                (Some(wake), Some(register)) => Some(wake.min(register)),
6906                (wake, register) => wake.or(register),
6907            };
6908            #[cfg(test)]
6909            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6910            let probe_sleep = supervisor_sleep(wake_at);
6911            tokio::pin!(probe_sleep);
6912            // A `protocol: "none"` child (a plain process that never registers
6913            // over the subc wire, such as nats-server) with no HTTP health
6914            // check has nothing to probe, so it waits only for its exit or a
6915            // supervisor command. A rescan that adds an HTTP check is a command.
6916            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6917            #[cfg(test)]
6918            {
6919                let registration_pending = wire_child
6920                    && registration_changes
6921                        .as_ref()
6922                        .is_some_and(|changes| changes.has_changed().unwrap());
6923                let mut state = lock_snapshot(&snapshot).unwrap();
6924                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6925                    generation: state.spawn_generation,
6926                    turn: state.actor_turns,
6927                    registered_connection: health_probe.registered_connection,
6928                    next_probe_at: health_probe.next_probe_at,
6929                    wake_after,
6930                });
6931            }
6932            let active_child = child.as_mut().expect("child checked above");
6933            tokio::select! {
6934                wait_result = active_child.wait() => {
6935                    if let Ok(mut state) = lock_snapshot(&snapshot) {
6936                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6937                    }
6938                    #[cfg(test)]
6939                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6940                    // Every arm below that gives up on the CHILD must keep the
6941                    // supervision task itself alive (child = None, loop
6942                    // continues into command-serving mode). Returning here
6943                    // closes the command channel, which makes the module
6944                    // permanently unrestartable in-band: a clean child exit
6945                    // of an enabled module once wedged the fleet this way
6946                    // ('supervisor command channel is closed') and required a
6947                    // full daemon restart to recover.
6948                    let exit_report = match wait_result {
6949                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6950                        Err(err) => {
6951                            active_child.drain_stderr(&spec.module_id).await;
6952                            fail_after_wait_error(
6953                                &spec.module_id,
6954                                &snapshot,
6955                                &runtime.terminal_ring,
6956                                &runtime.spawn_events,
6957                                format!("failed to wait for supervised module: {err}"),
6958                            &runtime.failure_context(true),
6959                            );
6960                            untrack_if_registration_released(
6961                                &process_liveness,
6962                                &registry,
6963                                &spec.module_id,
6964                                &snapshot,
6965                            );
6966                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6967                            child = None;
6968                            continue;
6969                        }
6970                    };
6971                    active_child.drain_stderr(&spec.module_id).await;
6972
6973                    let next = on_child_exit(
6974                        &spec,
6975                        runtime.restart_policy,
6976                        &registry,
6977                        &snapshot,
6978                        &runtime.terminal_ring,
6979                        &runtime.spawn_events,
6980                        &runtime.child_roster,
6981                        exit_report,
6982                        true,
6983                    ).await;
6984                    runtime.start_requester = None;
6985                    runtime.pending_activity_restart = false;
6986                    // The exit is recorded, so a daemon shutdown may stop
6987                    // waiting for this child (see `SupervisedChild::wait`).
6988                    active_child.release_roster();
6989                    match next {
6990                        NextAction::Stop { registration_released } => {
6991                            if registration_released {
6992                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6993                            }
6994                            child = None;
6995                        }
6996                        NextAction::Restart { schedule } => {
6997                            let delay = schedule.map_or(
6998                                runtime.restart_policy.delay_for_restart(0),
6999                                |schedule| schedule.delay,
7000                            );
7001                            if let Some(schedule) = schedule {
7002                                log_crash_respawn(&spec.module_id, schedule);
7003                            }
7004                            // The exited child is fully recorded at this point,
7005                            // so release it and count the backoff down in the
7006                            // command-serving branch below rather than sleeping
7007                            // here: commands cannot be received from inside this
7008                            // select arm, and an operator disable or drain that
7009                            // arrives during the backoff must cancel the pending
7010                            // respawn instead of waiting for it to spawn first.
7011                            child = None;
7012                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7013                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7014                        }
7015                    }
7016                }
7017                command = commands.recv() => {
7018                    #[cfg(test)]
7019                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7020                    let Some(command) = command else {
7021                        return;
7022                    };
7023                    if !handle_supervisor_command(
7024                        command,
7025                        &mut spec,
7026                        &mut runtime,
7027                        &registry,
7028                        &process_liveness,
7029                        &snapshot,
7030                        &mut child,
7031                        &mut commands,
7032                        &mut requeued,
7033                    ).await {
7034                        return;
7035                    }
7036                }
7037                _ = async {
7038                    match registration_changes.as_mut() {
7039                        Some(changes) => { let _ = changes.changed().await; }
7040                        None => std::future::pending().await,
7041                    }
7042                }, if wire_child => {
7043                    #[cfg(test)]
7044                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7045                }
7046                _ = &mut probe_sleep => {
7047                    #[cfg(test)]
7048                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7049                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7050                        // Recheck after the wake: HELLO, exit or shutdown may
7051                        // have become ready in the same executor tick.
7052                        active_child.registration_deadline = None;
7053                        if !runtime.child_roster.is_closed()
7054                            && !active_child.registered_since_spawn()
7055                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7056                        {
7057                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7058                                wait_secs = runtime.register_timeout.as_secs(),
7059                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7060                            active_child.spawn_failure = Some("registration_timeout".into());
7061                            if let Err(error) = stop_unregistered_child(active_child).await {
7062                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7063                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7064                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7065                                    &runtime.failure_context(true));
7066                                child = None;
7067                            }
7068                            // The ordinary exit handler records the child's exit,
7069                            // spends crash budget, and schedules recovery.
7070                            continue;
7071                        }
7072                    }
7073                    if health_probe.due() {
7074                        run_health_probe_cycle(
7075                            &spec,
7076                            &runtime,
7077                            &registry,
7078                            &process_liveness,
7079                            &snapshot,
7080                            &mut child,
7081                        ).await;
7082                        if child.is_none() {
7083                            runtime.start_requester = None;
7084                            runtime.pending_activity_restart = false;
7085                        }
7086                        if child.is_some() {
7087                            health_probe.schedule_next(&spec, runtime.health.cadence);
7088                        }
7089                    }
7090                }
7091            }
7092        } else if let Some(pending) = pending_respawn {
7093            tokio::select! {
7094                _ = sleep_until(pending.deadline) => {
7095                    pending_respawn = None;
7096                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7097                    // A command handled below while the backoff elapsed may
7098                    // have stopped the module; never respawn past an operator's
7099                    // disable or drain.
7100                    if !respawn_still_pending(&snapshot) {
7101                        continue;
7102                    }
7103                    // The daemon began shutting down during the backoff: the
7104                    // spawn would be refused anyway, and refusing it here
7105                    // leaves the module stopped instead of reporting a
7106                    // failed restart.
7107                    if runtime.child_roster.is_closed() {
7108                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7109                            state.state = ModuleState::Stopped;
7110                        });
7111                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7112                        continue;
7113                    }
7114                    if let Err(err) = release_dead_registration(
7115                        &registry,
7116                        runtime.forwarding.as_deref(),
7117                        &snapshot,
7118                        &spec.module_id,
7119                        &runtime.failure_context(runtime.recovery_is_daemon),
7120                    ).await {
7121                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7122                        continue;
7123                    }
7124
7125                    if matches!(pending.kind, RespawnKind::Reload) {
7126                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7127                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7128                        if result.is_ok() && runtime.pending_activity_restart {
7129                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7130                        }
7131                        runtime.pending_activity_restart = false;
7132                        if let Some(reply) = reply { let _ = reply.send(result); }
7133                        continue;
7134                    }
7135                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7136                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7137                        Ok(next_child) => {
7138                            child = Some(next_child);
7139                            if runtime.pending_activity_restart {
7140                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7141                                runtime.pending_activity_restart = false;
7142                            }
7143                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7144                        }
7145                        Err(err) => {
7146                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7147                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7148                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7149                        }
7150                    }
7151                }
7152                command = commands.recv() => {
7153                    let Some(command) = command else {
7154                        return;
7155                    };
7156                    if !handle_supervisor_command(
7157                        command,
7158                        &mut spec,
7159                        &mut runtime,
7160                        &registry,
7161                        &process_liveness,
7162                        &snapshot,
7163                        &mut child,
7164                        &mut commands,
7165                        &mut requeued,
7166                    ).await {
7167                        return;
7168                    }
7169                    // Reconcile the pending respawn with what the command did:
7170                    // a start may already have spawned a fresh child,
7171                    // while a disable or drain moved the snapshot out of the
7172                    // state the respawn was counting down from.
7173                    if child.is_some() || !respawn_still_pending(&snapshot) {
7174                        pending_respawn = None;
7175                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7176                    }
7177                }
7178            }
7179        } else {
7180            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7181                (state.enabled && state.state == ModuleState::Failed)
7182                    .then_some(state.failed_retry.pending)
7183                    .flatten()
7184            });
7185            let command = if lock_snapshot(&snapshot)
7186                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7187            {
7188                tokio::select! {
7189                    command = commands.recv() => command,
7190                    _ = previous_instance_recheck.tick() => {
7191                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7192                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7193                            if enabled && !runtime.child_roster.is_closed() {
7194                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7195                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7196                                    Ok(next_child) => child = Some(next_child),
7197                                    Err(error) => {
7198                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7199                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7200                                    }
7201                                }
7202                            } else {
7203                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7204                                    state.waiting_for_previous_pid = None;
7205                                    state.waiting_for_previous_detail = None;
7206                                });
7207                            }
7208                        }
7209                        continue;
7210                    }
7211                }
7212            } else {
7213                tokio::select! {
7214                    biased;
7215                    command = commands.recv() => command,
7216                    _ = async {
7217                        match slow_retry {
7218                            Some(retry) => sleep_until(retry.deadline).await,
7219                            None => std::future::pending().await,
7220                        }
7221                    } => {
7222                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7223                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7224                            state.failed_retry.pending = None;
7225                        });
7226                        if runtime.child_roster.is_closed() {
7227                            continue;
7228                        }
7229                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7230                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7231                        if let Err(error) = set_child_enabled(
7232                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7233                            &runtime.failure_context(true),
7234                        ).await {
7235                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7236                        }
7237                        continue;
7238                    }
7239                }
7240            };
7241            let Some(command) = command else {
7242                return;
7243            };
7244            if !handle_supervisor_command(
7245                command,
7246                &mut spec,
7247                &mut runtime,
7248                &registry,
7249                &process_liveness,
7250                &snapshot,
7251                &mut child,
7252                &mut commands,
7253                &mut requeued,
7254            )
7255            .await
7256            {
7257                return;
7258            }
7259        }
7260    }
7261}
7262
7263// Preserve the deadline until the future is polled. Starting a relative sleep
7264// after the executor yields would restart part of the countdown.
7265async fn supervisor_sleep(wake_at: Option<Instant>) {
7266    match wake_at {
7267        Some(deadline) => sleep_until(deadline).await,
7268        None => std::future::pending().await,
7269    }
7270}
7271
7272fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7273    info!(
7274        module_id,
7275        restart_in_window = schedule.restart_in_window,
7276        delay_ms = schedule.delay.as_millis() as u64,
7277        "respawning after crash"
7278    );
7279}
7280
7281/// Whether the respawn a backoff was counting down to is still wanted. A
7282/// disable or drain handled while the backoff elapsed moves the snapshot out
7283/// of `Restarting`, and the operator's stop must win over the pending respawn,
7284/// so every sleep-then-spawn path re-validates against the live snapshot
7285/// instead of assuming the state it left behind still holds.
7286fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7287    matches!(
7288        lock_snapshot(snapshot),
7289        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7290    )
7291}
7292
7293enum NextAction {
7294    Stop {
7295        registration_released: bool,
7296    },
7297    Restart {
7298        schedule: Option<CrashRestartSchedule>,
7299    },
7300}
7301
7302#[allow(clippy::too_many_arguments)]
7303async fn handle_supervisor_command(
7304    command: SupervisorCommand,
7305    spec: &mut ModuleSpec,
7306    runtime: &mut SupervisorRuntimeConfig,
7307    registry: &Arc<Registry>,
7308    process_liveness: &SupervisorProcessLiveness,
7309    snapshot: &SharedSnapshot,
7310    child: &mut Option<SupervisedChild>,
7311    commands: &mut mpsc::Receiver<SupervisorCommand>,
7312    requeued: &mut VecDeque<SupervisorCommand>,
7313) -> bool {
7314    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7315        runtime.recovery_is_daemon = false;
7316    }
7317    if matches!(
7318        command,
7319        SupervisorCommand::Drain { .. }
7320            | SupervisorCommand::Retire { .. }
7321            | SupervisorCommand::SetEnabled { .. }
7322            | SupervisorCommand::Restart { .. }
7323    ) {
7324        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7325            state.failed_retry.cancel()
7326        });
7327    }
7328    let command =
7329        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7330            match command {
7331                SupervisorCommand::Restart { reply, .. }
7332                | SupervisorCommand::Reload { reply, .. }
7333                | SupervisorCommand::Swap { reply, .. } => {
7334                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7335                    return true;
7336                }
7337                SupervisorCommand::SetEnabled {
7338                    enabled: true,
7339                    reply,
7340                    ..
7341                } => {
7342                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7343                    return true;
7344                }
7345                other => other,
7346            }
7347        } else {
7348            command
7349        };
7350    match command {
7351        SupervisorCommand::Drain { requester, reply } => {
7352            // A plain stop runs no forwarding drain, so nothing reaches the
7353            // module over its connection before the wait: ask by signal.
7354            let result = drain_optional_child(
7355                &spec.module_id,
7356                spec.protocol,
7357                StopNotice::NotSent,
7358                registry,
7359                runtime.forwarding.as_deref(),
7360                snapshot,
7361                &runtime.terminal_ring,
7362                &runtime.spawn_events,
7363                child,
7364                runtime.drain_timeout,
7365                ModuleState::Stopped,
7366                None,
7367                &FailureContext::operator(),
7368            )
7369            .await;
7370            let registration_released = result.is_ok();
7371            if registration_released {
7372                runtime.record_activity_transition(
7373                    "subc.module_stopped",
7374                    &spec.module_id,
7375                    requester,
7376                );
7377            }
7378            let _ = reply.send(result);
7379            if registration_released {
7380                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7381            }
7382            false
7383        }
7384        SupervisorCommand::Retire { requester, reply } => {
7385            let result = async {
7386                let stop_notice = begin_forwarding_drain_if_configured(
7387                    spec,
7388                    runtime,
7389                    registry,
7390                    snapshot,
7391                    None,
7392                    RouteCloseReason::Disable,
7393                )
7394                .await?;
7395                drain_optional_child(
7396                    &spec.module_id,
7397                    spec.protocol,
7398                    stop_notice,
7399                    registry,
7400                    runtime.forwarding.as_deref(),
7401                    snapshot,
7402                    &runtime.terminal_ring,
7403                    &runtime.spawn_events,
7404                    child,
7405                    runtime.drain_timeout,
7406                    ModuleState::Stopped,
7407                    None,
7408                    &FailureContext::operator(),
7409                )
7410                .await
7411            }
7412            .await;
7413            let registration_released = result.is_ok();
7414            if registration_released {
7415                runtime.record_activity_transition(
7416                    "subc.module_stopped",
7417                    &spec.module_id,
7418                    requester,
7419                );
7420            }
7421            let _ = reply.send(result);
7422            if registration_released {
7423                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7424            }
7425            false
7426        }
7427        SupervisorCommand::Restart {
7428            requester,
7429            drain_timeout_ms,
7430            received_at_generation,
7431            queued_at,
7432            reply,
7433        } => {
7434            // Without this line a restart that waited in the queue (behind a
7435            // health probe cycle or another command) was invisible: the log
7436            // showed only the drain timing out, minutes after the operator's call.
7437            info!(
7438                module_id = %spec.module_id,
7439                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7440                "restart command dequeued"
7441            );
7442            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7443            // caller whose own request lane rides the module being restarted: the
7444            // caller's in-flight request keeps the drain from quiescing, the drain
7445            // keeps the restart from completing, and the completion keeps the reply
7446            // from releasing the caller — so the drain always timed out and cut the
7447            // initiator with a GOODBYE, even on a healthy module. Replying once the
7448            // restart is validated lets a self-lane caller settle, which is exactly
7449            // what makes the drain succeed. Completion is observable via
7450            // supervisor.list / module status; a post-ack failure lands the module
7451            // in a visible terminal state below rather than in a reply nobody can
7452            // receive.
7453            let validation = match lock_snapshot(snapshot) {
7454                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7455                    module_id: spec.module_id.clone(),
7456                }),
7457                Ok(_) => Ok(()),
7458                Err(err) => Err(err),
7459            };
7460            let initiated = validation.is_ok();
7461            let _ = reply.send(validation);
7462            // A restart asks for a fresh process. Commands run one at a time,
7463            // so a restart queued behind another restart (two operator calls
7464            // in quick succession) is dequeued the moment the first one has
7465            // spawned its replacement -- before that process has sent HELLO.
7466            // Running it would drain and kill the process the first restart
7467            // just produced, which is the opposite of what both callers asked
7468            // for. If a process spawned after this request was received is
7469            // still supervised, the request is already satisfied. Not when the
7470            // configuration changed since that spawn: then the newer process
7471            // predates the spec this restart may exist to apply.
7472            // Generation publication can lag a child that is already serving
7473            // while its identity is recorded. Only process creation after request
7474            // receipt satisfies a restart, not publication after receipt.
7475            let satisfied_by_generation = if initiated
7476                && child
7477                    .as_ref()
7478                    .is_some_and(|child| child.spawned_at > queued_at)
7479            {
7480                lock_snapshot(snapshot).ok().and_then(|state| {
7481                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7482                })
7483            } else {
7484                None
7485            };
7486            let satisfied_by_pending = initiated
7487                && child.is_none()
7488                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7489                    // No child exists yet, so this replacement will necessarily
7490                    // be created after the queued request, unlike a published
7491                    // generation whose process may already have been serving.
7492                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7493                    if pending {
7494                        state.coalesced_restart_pending = true;
7495                    }
7496                    pending
7497                });
7498            if satisfied_by_pending {
7499                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7500            } else if let Some(generation) = satisfied_by_generation {
7501                info!(
7502                    module_id = %spec.module_id,
7503                    received_at_generation,
7504                    "restart already satisfied by generation {generation}; not restarting again"
7505                );
7506            } else if initiated {
7507                // Precedence: this restart's operator override, else the module's
7508                // configured budget (already resolved into the runtime).
7509                let drain_timeout = drain_timeout_ms
7510                    .map(Duration::from_millis)
7511                    .unwrap_or(runtime.drain_timeout);
7512                if let Err(err) = restart_child(
7513                    spec,
7514                    runtime,
7515                    registry,
7516                    process_liveness,
7517                    snapshot,
7518                    child,
7519                    drain_timeout,
7520                )
7521                .await
7522                {
7523                    warn!(
7524                        module_id = %spec.module_id,
7525                        error = %err,
7526                        "operator restart failed after initiation ack; module state carries the outcome"
7527                    );
7528                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7529                        state.state = ModuleState::Failed;
7530                        clear_current_process_facts(state);
7531                    });
7532                } else {
7533                    runtime.start_requester = requester;
7534                    runtime.pending_activity_restart = true;
7535                }
7536            }
7537            true
7538        }
7539        SupervisorCommand::Reload { requester, reply } => {
7540            let result =
7541                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7542            if result.is_ok() {
7543                runtime.start_requester = requester;
7544                runtime.pending_activity_restart = true;
7545            }
7546            if result.is_ok()
7547                && runtime
7548                    .scheduled_respawn
7549                    .lock()
7550                    .unwrap_or_else(|p| p.into_inner())
7551                    .as_ref()
7552                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7553            {
7554                *runtime
7555                    .deferred_reload_reply
7556                    .lock()
7557                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7558            } else {
7559                let _ = reply.send(result);
7560            }
7561            true
7562        }
7563        SupervisorCommand::SetEnabled {
7564            enabled,
7565            requester,
7566            reply,
7567        } => {
7568            let previous_requester = runtime.start_requester.clone();
7569            if enabled {
7570                runtime.start_requester = requester.clone();
7571            }
7572            let result = set_child_enabled(
7573                spec,
7574                runtime,
7575                registry,
7576                process_liveness,
7577                snapshot,
7578                child,
7579                enabled,
7580                &runtime.failure_context(false),
7581            )
7582            .await;
7583            if matches!(result, Ok(true)) {
7584                runtime.pending_activity_restart = false;
7585                if enabled {
7586                    runtime.start_requester = requester;
7587                } else {
7588                    runtime.start_requester = None;
7589                    runtime.record_activity_transition(
7590                        "subc.module_disabled",
7591                        &spec.module_id,
7592                        requester,
7593                    );
7594                }
7595            } else {
7596                runtime.start_requester = previous_requester;
7597            }
7598            let _ = reply.send(result);
7599            true
7600        }
7601        SupervisorCommand::UpdateConfiguration {
7602            spec: next_spec,
7603            health,
7604            drain_timeout_ms,
7605            reply,
7606        } => {
7607            if let Some(handle) = &runtime.supervisor_handle {
7608                handle.apply_identity_configuration(&next_spec);
7609            }
7610            *spec = next_spec;
7611            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7612                state.configuration_updated_since_spawn = true;
7613            });
7614            let health_changed = runtime.health != health;
7615            runtime.health = health;
7616            // Reset the cadence and old endpoint's failure streak on a live
7617            // health-policy change rather than waiting for its old deadline.
7618            if health_changed {
7619                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7620                    state.health = ModuleHealthStatus::default();
7621                });
7622            }
7623            runtime.drain_timeout = drain_timeout_ms
7624                .map(Duration::from_millis)
7625                .unwrap_or(runtime.default_drain_timeout);
7626            *runtime
7627                .effective_drain_timeout
7628                .lock()
7629                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7630            let _ = reply.send(());
7631            true
7632        }
7633        SupervisorCommand::Swap {
7634            requester,
7635            ready_timeout,
7636            reply,
7637        } => {
7638            let mut swap_runtime = runtime.clone();
7639            swap_runtime.start_requester = requester;
7640            let before = lock_snapshot(snapshot)
7641                .ok()
7642                .map(|state| state.spawn_generation);
7643            let end = swap::run_swap(
7644                spec,
7645                &swap_runtime,
7646                registry,
7647                process_liveness,
7648                snapshot,
7649                child,
7650                commands,
7651                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7652                reply,
7653            )
7654            .await;
7655            if before
7656                != lock_snapshot(snapshot)
7657                    .ok()
7658                    .map(|state| state.spawn_generation)
7659            {
7660                runtime.start_requester = swap_runtime.start_requester;
7661            }
7662            requeued.extend(end.requeue);
7663            true
7664        }
7665    }
7666}
7667
7668async fn restart_child(
7669    spec: &ModuleSpec,
7670    runtime: &SupervisorRuntimeConfig,
7671    registry: &Registry,
7672    process_liveness: &SupervisorProcessLiveness,
7673    snapshot: &SharedSnapshot,
7674    child: &mut Option<SupervisedChild>,
7675    drain_timeout: Duration,
7676) -> Result<(), SuperviseError> {
7677    // Restart cycles a running module; it must not silently start a disabled one.
7678    if !lock_snapshot(snapshot)?.enabled {
7679        return Err(SuperviseError::Disabled {
7680            module_id: spec.module_id.clone(),
7681        });
7682    }
7683    let stop_notice = begin_forwarding_drain_with_timeout(
7684        spec,
7685        runtime,
7686        registry,
7687        snapshot,
7688        None,
7689        RouteCloseReason::Restart,
7690        drain_timeout,
7691    )
7692    .await?;
7693
7694    if child.is_some() {
7695        drain_optional_child(
7696            &spec.module_id,
7697            spec.protocol,
7698            stop_notice,
7699            registry,
7700            runtime.forwarding.as_deref(),
7701            snapshot,
7702            &runtime.terminal_ring,
7703            &runtime.spawn_events,
7704            child,
7705            drain_timeout,
7706            ModuleState::Restarting,
7707            Some(true),
7708            &FailureContext::operator(),
7709        )
7710        .await?;
7711    } else {
7712        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7713            state.enabled = true;
7714            state.state = ModuleState::Restarting;
7715            clear_current_process_facts(state);
7716        })?;
7717        release_dead_registration(
7718            registry,
7719            runtime.forwarding.as_deref(),
7720            snapshot,
7721            &spec.module_id,
7722            &runtime.failure_context(false),
7723        )
7724        .await?;
7725    }
7726
7727    reset_restart_count(snapshot, &spec.module_id)?;
7728    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7729    schedule_respawn(
7730        runtime,
7731        snapshot,
7732        &spec.module_id,
7733        runtime.restart_policy.backoff,
7734        RespawnKind::Spawn,
7735        false,
7736    )
7737}
7738
7739async fn reload_child(
7740    spec: &ModuleSpec,
7741    runtime: &SupervisorRuntimeConfig,
7742    registry: &Registry,
7743    process_liveness: &SupervisorProcessLiveness,
7744    snapshot: &SharedSnapshot,
7745    child: &mut Option<SupervisedChild>,
7746) -> Result<(), SuperviseError> {
7747    // Reload cycles a running module; it must not silently start a disabled one.
7748    if !lock_snapshot(snapshot)?.enabled {
7749        return Err(SuperviseError::Disabled {
7750            module_id: spec.module_id.clone(),
7751        });
7752    }
7753    let stop_notice = begin_forwarding_drain(
7754        spec,
7755        runtime,
7756        registry,
7757        snapshot,
7758        Some(true),
7759        RouteCloseReason::Reload,
7760    )
7761    .await?;
7762
7763    if child.is_some() {
7764        drain_optional_child(
7765            &spec.module_id,
7766            spec.protocol,
7767            stop_notice,
7768            registry,
7769            runtime.forwarding.as_deref(),
7770            snapshot,
7771            &runtime.terminal_ring,
7772            &runtime.spawn_events,
7773            child,
7774            runtime.drain_timeout,
7775            ModuleState::Restarting,
7776            Some(true),
7777            &FailureContext::operator(),
7778        )
7779        .await?;
7780    } else {
7781        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7782            state.enabled = true;
7783            state.state = ModuleState::Restarting;
7784            clear_current_process_facts(state);
7785        })?;
7786        release_dead_registration(
7787            registry,
7788            runtime.forwarding.as_deref(),
7789            snapshot,
7790            &spec.module_id,
7791            &runtime.failure_context(false),
7792        )
7793        .await?;
7794    }
7795
7796    reset_restart_count(snapshot, &spec.module_id)?;
7797    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7798    schedule_respawn(
7799        runtime,
7800        snapshot,
7801        &spec.module_id,
7802        runtime.restart_policy.backoff,
7803        RespawnKind::Reload,
7804        false,
7805    )
7806}
7807
7808async fn finish_reload_child(
7809    spec: &ModuleSpec,
7810    runtime: &SupervisorRuntimeConfig,
7811    registry: &Registry,
7812    process_liveness: &SupervisorProcessLiveness,
7813    snapshot: &SharedSnapshot,
7814    child: &mut Option<SupervisedChild>,
7815) -> Result<(), SuperviseError> {
7816    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7817    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7818        Ok(next_child) => next_child,
7819        Err(err) => {
7820            return handle_reload_spawn_failure(
7821                spec,
7822                runtime,
7823                process_liveness,
7824                snapshot,
7825                child,
7826                format!("new child failed to spawn: {err}"),
7827            )
7828            .await;
7829        }
7830    };
7831    *child = Some(next_child);
7832
7833    let wait_outcome = {
7834        let active_child = child.as_mut().expect("new reload child was just stored");
7835        wait_for_registration_after_reload(
7836            registry,
7837            &spec.module_id,
7838            snapshot,
7839            active_child,
7840            REGISTRY_RELEASE_TIMEOUT,
7841        )
7842        .await?
7843    };
7844
7845    match wait_outcome {
7846        RegistrationWaitOutcome::Registered => {
7847            child
7848                .as_mut()
7849                .expect("registered reload child")
7850                .registration_deadline = None;
7851            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7852            Ok(())
7853        }
7854        RegistrationWaitOutcome::Exited(exit_report) => {
7855            if let Some(active_child) = child.as_mut() {
7856                active_child.drain_stderr(&spec.module_id).await;
7857            }
7858            // Keep the reaped child's roster guard until its terminal is written.
7859            // Shutdown waits on that guard, not on the child Option used for respawn.
7860            let mut exited_child = child.take().expect("exited reload child is still stored");
7861            #[cfg(test)]
7862            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7863                gate.reached.notify_one();
7864                gate.resume.notified().await;
7865            }
7866            let result = handle_reload_child_registration_failure(
7867                spec,
7868                runtime,
7869                registry,
7870                process_liveness,
7871                snapshot,
7872                child,
7873                ReloadRegistrationFailure {
7874                    exit_report: registration_failure_exit_report(exit_report),
7875                    reason: exited_child
7876                        .spawn_failure
7877                        .clone()
7878                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7879                },
7880            )
7881            .await;
7882            exited_child.release_roster();
7883            result
7884        }
7885        RegistrationWaitOutcome::TimedOut => {
7886            let mut timed_out_child = child
7887                .take()
7888                .expect("timed-out reload child is still running");
7889            timed_out_child
7890                .start_kill()
7891                .map_err(|source| SuperviseError::Kill {
7892                    module_id: spec.module_id.clone(),
7893                    source,
7894                })?;
7895            let status = timed_out_child
7896                .wait()
7897                .await
7898                .map_err(|source| SuperviseError::Wait {
7899                    module_id: spec.module_id.clone(),
7900                    source,
7901                })?;
7902            timed_out_child.drain_stderr(&spec.module_id).await;
7903            handle_reload_child_registration_failure(
7904                spec,
7905                runtime,
7906                registry,
7907                process_liveness,
7908                snapshot,
7909                child,
7910                ReloadRegistrationFailure {
7911                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7912                        snapshot,
7913                        &timed_out_child,
7914                        &status,
7915                    )),
7916                    reason: format!(
7917                        "new child did not register within {:?}",
7918                        REGISTRY_RELEASE_TIMEOUT
7919                    ),
7920                },
7921            )
7922            .await
7923        }
7924    }
7925}
7926
7927#[allow(clippy::too_many_arguments)]
7928async fn set_child_enabled(
7929    spec: &ModuleSpec,
7930    runtime: &SupervisorRuntimeConfig,
7931    registry: &Registry,
7932    process_liveness: &SupervisorProcessLiveness,
7933    snapshot: &SharedSnapshot,
7934    child: &mut Option<SupervisedChild>,
7935    enabled: bool,
7936    failure: &FailureContext,
7937) -> Result<bool, SuperviseError> {
7938    let (current_enabled, current_state, respawn_pending) = {
7939        let state = lock_snapshot(snapshot)?;
7940        (state.enabled, state.state, state.respawn_pending)
7941    };
7942    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7943    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7944    // clean (Stopped) has no live process and no other in-band recovery — the
7945    // operator's start is the explicit recovery act and resets the budget. Without
7946    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7947    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7948    // the one providing every agent's shell.
7949    let revive_terminal = enabled
7950        && current_enabled
7951        && child.is_none()
7952        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7953            || (current_state == ModuleState::Restarting && !respawn_pending));
7954    if current_enabled == enabled && !revive_terminal {
7955        return Ok(false);
7956    }
7957
7958    if enabled {
7959        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7960            state.enabled = true;
7961            state.state = ModuleState::Starting;
7962            clear_current_process_facts(state);
7963        })?;
7964        #[cfg(test)]
7965        if runtime.test_seed_stale_facts_before_enable_spawn {
7966            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7967                state.process_alive = true;
7968                state.pid = Some(41);
7969                state.spawned_at_ms = Some(42);
7970                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7971                state.spawned_file_identity = Some(SpawnedFileIdentity {
7972                    device: 43,
7973                    inode: 44,
7974                    #[cfg(windows)]
7975                    windows_image: None,
7976                });
7977            })?;
7978        }
7979        release_dead_registration(
7980            registry,
7981            runtime.forwarding.as_deref(),
7982            snapshot,
7983            &spec.module_id,
7984            failure,
7985        )
7986        .await?;
7987        reset_restart_count(snapshot, &spec.module_id)?;
7988        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7989        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7990            Ok(next_child) => next_child,
7991            Err(err) => {
7992                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7993                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7994                return Err(err);
7995            }
7996        };
7997        *child = Some(next_child);
7998        debug!(module_id = %spec.module_id, "supervised module enabled");
7999        Ok(true)
8000    } else {
8001        let stop_notice = begin_forwarding_drain_if_configured(
8002            spec,
8003            runtime,
8004            registry,
8005            snapshot,
8006            Some(false),
8007            RouteCloseReason::Disable,
8008        )
8009        .await?;
8010        drain_optional_child(
8011            &spec.module_id,
8012            spec.protocol,
8013            stop_notice,
8014            registry,
8015            runtime.forwarding.as_deref(),
8016            snapshot,
8017            &runtime.terminal_ring,
8018            &runtime.spawn_events,
8019            child,
8020            runtime.drain_timeout,
8021            ModuleState::Disabled,
8022            Some(false),
8023            failure,
8024        )
8025        .await?;
8026        debug!(module_id = %spec.module_id, "supervised module disabled");
8027        Ok(true)
8028    }
8029}
8030
8031#[allow(clippy::too_many_arguments)]
8032async fn on_child_exit(
8033    spec: &ModuleSpec,
8034    policy: RestartPolicy,
8035    registry: &Registry,
8036    snapshot: &SharedSnapshot,
8037    terminal_ring: &Arc<Mutex<TerminalRing>>,
8038    spawn_events: &SpawnEventFeed,
8039    roster: &ChildRoster,
8040    exit_report: ExitReport,
8041    daemon_initiated: bool,
8042) -> NextAction {
8043    // Once the daemon has begun shutting down, no exit is a crash to recover
8044    // from: the module is exiting because the daemon is going away (EOF on its
8045    // connection, or a service manager signalling the whole cgroup). Record it
8046    // as such and never schedule a respawn, which would only start a process
8047    // for the shutdown to end again.
8048    if roster.is_closed() {
8049        return on_child_exit_during_daemon_shutdown(
8050            spec,
8051            registry,
8052            snapshot,
8053            terminal_ring,
8054            spawn_events,
8055            exit_report,
8056        )
8057        .await;
8058    }
8059    // Deliberate stops the supervisor asks for (operator stop, disable,
8060    // restart, reload, swap, a health restart, a drain that runs out of budget)
8061    // takes the child out of the supervise loop and reaps it in
8062    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8063    // that reaches this point is either unrequested or a first-HELLO timeout,
8064    // which is deliberately classified as a crash even after a clean TERM exit.
8065    //
8066    // For a subc-wire module a clean exit is still a stop: those modules are
8067    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8068    // crash, and exiting 0 is a deliberate choice the module made. A
8069    // `protocol: "none"` module is a stock program we cannot change, and many
8070    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8071    // stop would leave the module down for good after any stray signal, so it
8072    // goes through the crash path instead: it spends restart budget, respawns
8073    // with the crash backoff, and ends `failed` when the budget runs out.
8074    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8075        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8076    match exit_report.kind {
8077        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8078            info!(
8079                module_id = %spec.module_id,
8080                exit_code = ?exit_report.code,
8081                exit_signal = ?exit_report.signal,
8082                "supervised module exited cleanly"
8083            );
8084            if let Err(err) = publish_after_terminal_record(
8085                &spec.module_id,
8086                || {
8087                    record_terminal(
8088                        &spec.module_id,
8089                        terminal_ring,
8090                        spawn_events,
8091                        &exit_report,
8092                        TerminalDisposition::Stopped,
8093                    )
8094                },
8095                || {
8096                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8097                        state.state = ModuleState::Stopped;
8098                        clear_current_process_facts(state);
8099                        state.last_exit = Some(exit_report.clone());
8100                    })
8101                },
8102            ) {
8103                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8104            }
8105            let registration_released = match wait_for_registration_release(
8106                registry,
8107                &spec.module_id,
8108                REGISTRY_RELEASE_TIMEOUT,
8109            )
8110            .await
8111            {
8112                Ok(()) => true,
8113                Err(err) => {
8114                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8115                    false
8116                }
8117            };
8118            NextAction::Stop {
8119                registration_released,
8120            }
8121        }
8122        ExitKind::Clean | ExitKind::Crash => {
8123            if unrequested_clean_exit_of_protocol_none {
8124                warn!(
8125                    module_id = %spec.module_id,
8126                    exit_code = ?exit_report.code,
8127                    exit_signal = ?exit_report.signal,
8128                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8129                );
8130            } else {
8131                warn!(
8132                    module_id = %spec.module_id,
8133                    exit_code = ?exit_report.code,
8134                    exit_signal = ?exit_report.signal,
8135                    "supervised module exited abnormally (crash)"
8136                );
8137            }
8138            let mut restart_schedule = None;
8139            let mut disposition = TerminalDisposition::Disabled;
8140            let mut final_state = ModuleState::Disabled;
8141            // Set only when the budget is what stopped the module, so the
8142            // terminal record says which limit was hit rather than leaving
8143            // `failed` to be read as "crashed once, badly".
8144            let mut disposition_detail = lock_snapshot(snapshot)
8145                .ok()
8146                .and_then(|mut state| state.spawn_failure.take());
8147            let now = Instant::now();
8148            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8149                if state.enabled {
8150                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8151                        // Reserving the last slot exhausts the budget for the
8152                        // replacement, not for this exit. Publish its scheduled
8153                        // recovery atomically with the reservation so connection
8154                        // cleanup sees it while terminal recording still holds
8155                        // the visible state at Running.
8156                        state.respawn_pending = true;
8157                        final_state = ModuleState::Restarting;
8158                        restart_schedule = Some(schedule);
8159                        disposition = TerminalDisposition::Restarting;
8160                    } else {
8161                        final_state = ModuleState::Failed;
8162                        disposition = TerminalDisposition::Failed;
8163                        let budget = policy.budget_exhausted_detail();
8164                        disposition_detail =
8165                            Some(disposition_detail.take().map_or_else(
8166                                || budget.clone(),
8167                                |cause| format!("{cause}; {budget}"),
8168                            ));
8169                    }
8170                } else {
8171                    disposition = TerminalDisposition::Disabled;
8172                }
8173            }) {
8174                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8175                return NextAction::Stop {
8176                    registration_released: false,
8177                };
8178            }
8179            if disposition == TerminalDisposition::Failed {
8180                // The window is in the message, not only in the fields: this line
8181                // is read in a scrollback where a bare `max_restarts=3` reads as a
8182                // lifetime cap and sends the operator looking for three crashes
8183                // that never happened together.
8184                error!(
8185                    module_id = %spec.module_id,
8186                    max_restarts = policy.max_restarts,
8187                    window_secs = policy.window.as_secs(),
8188                    "module stopped: {}",
8189                    policy.budget_exhausted_detail()
8190                );
8191            }
8192            if let Err(err) = publish_after_terminal_record(
8193                &spec.module_id,
8194                || {
8195                    record_terminal_with_detail(
8196                        &spec.module_id,
8197                        terminal_ring,
8198                        spawn_events,
8199                        &exit_report,
8200                        disposition,
8201                        disposition_detail,
8202                    )
8203                },
8204                || {
8205                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8206                        state.state = final_state;
8207                        clear_current_process_facts(state);
8208                        if final_state == ModuleState::Failed
8209                            && daemon_initiated
8210                            && !roster.is_closed()
8211                        {
8212                            state.failed_retry.schedule(&spec.module_id);
8213                        }
8214                        state.last_exit = Some(exit_report.clone());
8215                    })
8216                },
8217            ) {
8218                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8219            }
8220
8221            if let Some(schedule) = restart_schedule {
8222                NextAction::Restart {
8223                    schedule: Some(schedule),
8224                }
8225            } else {
8226                let registration_released = match wait_for_registration_release(
8227                    registry,
8228                    &spec.module_id,
8229                    REGISTRY_RELEASE_TIMEOUT,
8230                )
8231                .await
8232                {
8233                    Ok(()) => true,
8234                    Err(err) => {
8235                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8236                        false
8237                    }
8238                };
8239                NextAction::Stop {
8240                    registration_released,
8241                }
8242            }
8243        }
8244        ExitKind::DeliberateSeverance => {
8245            warn!(
8246                module_id = %spec.module_id,
8247                exit_code = ?exit_report.code,
8248                exit_signal = ?exit_report.signal,
8249                "supervised module exited after deliberate connection severance"
8250            );
8251            let should_restart = match lock_snapshot(snapshot) {
8252                Ok(state) => state.enabled,
8253                Err(err) => {
8254                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8255                    return NextAction::Stop {
8256                        registration_released: false,
8257                    };
8258                }
8259            };
8260            let final_state = if should_restart {
8261                ModuleState::Restarting
8262            } else {
8263                ModuleState::Disabled
8264            };
8265            if let Err(err) = publish_after_terminal_record(
8266                &spec.module_id,
8267                || {
8268                    record_terminal(
8269                        &spec.module_id,
8270                        terminal_ring,
8271                        spawn_events,
8272                        &exit_report,
8273                        terminal_disposition(final_state),
8274                    )
8275                },
8276                || {
8277                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8278                        state.state = final_state;
8279                        clear_current_process_facts(state);
8280                        state.last_exit = Some(exit_report.clone());
8281                        state.lifetime_restarts += 1;
8282                    })
8283                },
8284            ) {
8285                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8286                return NextAction::Stop {
8287                    registration_released: false,
8288                };
8289            }
8290
8291            if should_restart {
8292                NextAction::Restart { schedule: None }
8293            } else {
8294                let registration_released = match wait_for_registration_release(
8295                    registry,
8296                    &spec.module_id,
8297                    REGISTRY_RELEASE_TIMEOUT,
8298                )
8299                .await
8300                {
8301                    Ok(()) => true,
8302                    Err(err) => {
8303                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8304                        false
8305                    }
8306                };
8307                NextAction::Stop {
8308                    registration_released,
8309                }
8310            }
8311        }
8312    }
8313}
8314
8315async fn on_child_exit_during_daemon_shutdown(
8316    spec: &ModuleSpec,
8317    registry: &Registry,
8318    snapshot: &SharedSnapshot,
8319    terminal_ring: &Arc<Mutex<TerminalRing>>,
8320    spawn_events: &SpawnEventFeed,
8321    exit_report: ExitReport,
8322) -> NextAction {
8323    info!(
8324        module_id = %spec.module_id,
8325        exit_code = ?exit_report.code,
8326        exit_signal = ?exit_report.signal,
8327        exit_kind = ?exit_report.kind,
8328        "supervised module exited during daemon shutdown; not restarting it"
8329    );
8330    if let Err(err) = publish_after_terminal_record(
8331        &spec.module_id,
8332        || {
8333            record_terminal(
8334                &spec.module_id,
8335                terminal_ring,
8336                spawn_events,
8337                &exit_report,
8338                TerminalDisposition::DaemonShutdown,
8339            )
8340        },
8341        || {
8342            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8343                state.state = ModuleState::Stopped;
8344                clear_current_process_facts(state);
8345                state.last_exit = Some(exit_report.clone());
8346            })
8347        },
8348    ) {
8349        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8350    }
8351    let registration_released =
8352        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8353            .await
8354            .is_ok();
8355    NextAction::Stop {
8356        registration_released,
8357    }
8358}
8359
8360// Keep the status lock out of terminal I/O. Readers may observe the previous
8361// state while recording is in progress, but never a completed exit without its
8362// history. Journal errors are logged by the journal and retained in the ring;
8363// a recording panic still publishes the final state before resuming the unwind.
8364fn publish_after_terminal_record<T>(
8365    module_id: &str,
8366    record: impl FnOnce(),
8367    publish: impl FnOnce() -> T,
8368) -> T {
8369    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8370    let published = publish();
8371    if let Err(panic) = recorded {
8372        error!(
8373            module_id,
8374            "terminal recording panicked; published final state before resuming unwind"
8375        );
8376        std::panic::resume_unwind(panic);
8377    }
8378    published
8379}
8380
8381fn fail_after_wait_error(
8382    module_id: &str,
8383    snapshot: &SharedSnapshot,
8384    terminal_ring: &Arc<Mutex<TerminalRing>>,
8385    spawn_events: &SpawnEventFeed,
8386    detail: String,
8387    failure: &FailureContext,
8388) {
8389    let report = wait_error_exit_report();
8390    publish_after_terminal_record(
8391        module_id,
8392        || {
8393            record_terminal_with_detail(
8394                module_id,
8395                terminal_ring,
8396                spawn_events,
8397                &report,
8398                TerminalDisposition::Failed,
8399                Some(detail),
8400            )
8401        },
8402        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8403    );
8404}
8405
8406#[cfg(test)]
8407fn record_wait_error_terminal(
8408    module_id: &str,
8409    terminal_ring: &Arc<Mutex<TerminalRing>>,
8410    spawn_events: &SpawnEventFeed,
8411) {
8412    record_terminal(
8413        module_id,
8414        terminal_ring,
8415        spawn_events,
8416        &wait_error_exit_report(),
8417        TerminalDisposition::Failed,
8418    );
8419}
8420
8421fn record_terminal(
8422    module_id: &str,
8423    terminal_ring: &Arc<Mutex<TerminalRing>>,
8424    spawn_events: &SpawnEventFeed,
8425    exit_report: &ExitReport,
8426    disposition: TerminalDisposition,
8427) {
8428    record_terminal_with_detail(
8429        module_id,
8430        terminal_ring,
8431        spawn_events,
8432        exit_report,
8433        disposition,
8434        None,
8435    );
8436}
8437
8438/// The ring lock is held only to capture the read (see
8439/// `TerminalJournal::capture_read`), so this module's exits keep recording
8440/// while the journal files are read. Blocking: it reads files.
8441fn durable_terminal_history_of(
8442    terminal_ring: &Mutex<TerminalRing>,
8443    module_id: &str,
8444) -> subc_control::TerminalHistory {
8445    let read = terminal_ring
8446        .lock()
8447        .unwrap_or_else(|p| p.into_inner())
8448        .capture_durable_history();
8449    read.read(module_id)
8450}
8451
8452fn record_terminal_with_detail(
8453    module_id: &str,
8454    terminal_ring: &Arc<Mutex<TerminalRing>>,
8455    spawn_events: &SpawnEventFeed,
8456    exit_report: &ExitReport,
8457    disposition: TerminalDisposition,
8458    disposition_detail: Option<String>,
8459) {
8460    #[cfg(test)]
8461    terminal_history_tests::before_terminal_record(spawn_events);
8462    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8463    let record = TerminalRecord {
8464        exit_code: exit_report.code,
8465        exit_signal: exit_report.signal,
8466        at_ms: exit_report.at_ms,
8467        disposition,
8468        exit_kind: exit_report.kind.into(),
8469        disposition_detail,
8470    };
8471    terminal_ring
8472        .lock()
8473        .unwrap_or_else(|poisoned| poisoned.into_inner())
8474        .record_exit(module_id, record);
8475}
8476
8477fn untrack_if_registration_released(
8478    process_liveness: &SupervisorProcessLiveness,
8479    registry: &Registry,
8480    module_id: &str,
8481    snapshot: &SharedSnapshot,
8482) {
8483    match registry.get_module(module_id) {
8484        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8485        Ok(Some(_)) => {}
8486        Err(err) => {
8487            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8488        }
8489    }
8490}
8491
8492/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8493/// then apply the module's configured entries minus daemon-private capture keys.
8494///
8495/// Separated from `spawn_child` only so it can be asserted without spawning a
8496/// process — a duplicate of this logic in a test would pass while the real one
8497/// drifted, which is the defect class this function exists to avoid.
8498/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8499/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8500/// either and the argument would stop a stock binary from starting at all.
8501/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8502///
8503/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8504/// through [`apply_wire_spawn_args_for_role`].
8505#[cfg(test)]
8506fn apply_wire_spawn_args(
8507    command: &mut Command,
8508    spec: &ModuleSpec,
8509    connection_file_path: Option<&std::path::Path>,
8510    handle: Option<&SupervisorHandle>,
8511) -> Result<Option<NonceHandoff>, SuperviseError> {
8512    apply_wire_spawn_args_for_role(
8513        command,
8514        spec,
8515        connection_file_path,
8516        handle,
8517        SpawnRole::Plain,
8518    )
8519}
8520
8521/// The read end of a spawn's launch-nonce pipe, prepared by
8522/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8523/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8524/// server before spawn and authenticates its client after the child exists.
8525#[cfg(unix)]
8526type NonceHandoff = subc_os::LaunchNonceHandoff;
8527#[cfg(windows)]
8528type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8529#[cfg(not(any(unix, windows)))]
8530type NonceHandoff = std::convert::Infallible;
8531
8532/// Create the launch nonce that a new module process must present when it
8533/// registers or opens a consumer connection, including a replacement process
8534/// started alongside a still-serving module.
8535///
8536/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8537/// a separate candidate token so the still-serving incumbent and its consumers
8538/// keep their nonce. Both records are installed before the process exists, so
8539/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8540///
8541/// On Unix the nonce is delivered only through a pipe. It is written into
8542/// a pipe whose read end the child gets as descriptor 3, named by
8543/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8544/// process of the same user cannot read it with `ps eww`. That handoff is
8545/// returned rather than installed here, because installing it replaces
8546/// whatever the child has at descriptor 3 and so must be the last pre-exec
8547/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8548/// the Linux callback that writes the process into its cgroup. Only after that
8549/// does `handoff.install_last` register the descriptor-3 replacement, so it
8550/// cannot close a descriptor the cgroup callback still needs.
8551/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8552fn apply_wire_spawn_args_for_role(
8553    command: &mut Command,
8554    spec: &ModuleSpec,
8555    connection_file_path: Option<&std::path::Path>,
8556    handle: Option<&SupervisorHandle>,
8557    role: SpawnRole,
8558) -> Result<Option<NonceHandoff>, SuperviseError> {
8559    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8560    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8561    // included: a daemon started from a module's process tree inherits it,
8562    // and passing it on would point the child at a descriptor it does not
8563    // have.
8564    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8565    #[cfg(windows)]
8566    {
8567        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8568        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8569    }
8570    // Remove inherited or configured copies too: withholding must mean absent.
8571    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8572    if spec.protocol == ModuleProtocol::None {
8573        return Ok(None);
8574    }
8575    if let Some(connection_file_path) = connection_file_path {
8576        command.arg(SUBC_ARG).arg(connection_file_path);
8577    }
8578
8579    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8580    // route.open attestation. Reserved modules additionally use the same nonce
8581    // for HELLO id-squatting protection. A respawn rotates both records.
8582    let nonce = generate_launch_nonce()?;
8583    if let Some(handle) = handle {
8584        match role {
8585            SpawnRole::Plain => {
8586                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8587                if spec.reserved {
8588                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8589                }
8590            }
8591            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8592        }
8593    }
8594    #[cfg(unix)]
8595    let handoff = {
8596        let handoff =
8597            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8598                program: spec.program.clone(),
8599                source,
8600                cgroup_path: None,
8601            })?;
8602        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8603        Some(handoff)
8604    };
8605    #[cfg(windows)]
8606    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8607    #[cfg(not(any(unix, windows)))]
8608    let handoff = None;
8609    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8610    // until the daemon's per-spawn delivery records show that every running
8611    // Windows module has read its secret from the named pipe.
8612    #[cfg(not(unix))]
8613    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8614    Ok(handoff)
8615}
8616
8617fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8618    command.env_remove(CK_LOG_ENV);
8619    // The spawn role is the supervisor's to set, and only on a swap candidate
8620    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8621    // it, is what makes it absent on a plain spawn: the daemon's own
8622    // environment could carry it, and so could a spec built outside daemon
8623    // config (config refuses it as an `env` key). A module reading it on a
8624    // plain restart would pick the long swap budget and leave callers waiting.
8625    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8626    for (key, value) in &spec.env {
8627        // cortexkit-log currently exposes retention only as a Rust struct, not
8628        // environment names. These values are daemon-private sink metadata and
8629        // must never become a public child-process contract by being inherited.
8630        if matches!(
8631            key.as_str(),
8632            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8633        ) || key == SUBC_SPAWN_ROLE_ENV
8634        {
8635            continue;
8636        }
8637        command.env(key, value);
8638    }
8639}
8640
8641#[cfg(windows)]
8642fn prepare_windows_nonce_pipe(
8643    command: &mut Command,
8644    spec: &ModuleSpec,
8645    nonce: &str,
8646) -> Option<NonceHandoff> {
8647    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8648        Ok(handoff) => {
8649            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8650            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8651            // still register until every module reads the pipe. A reader without
8652            // this variable set to env returns the pipe error instead of using
8653            // an inherited or configured environment copy.
8654            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8655            Some(handoff)
8656        }
8657        Err(error) => {
8658            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8659            None
8660        }
8661    }
8662}
8663
8664#[cfg(windows)]
8665fn start_windows_nonce_pipe(
8666    handoff: Option<NonceHandoff>,
8667    pid: u32,
8668    spec: &ModuleSpec,
8669) -> Option<subc_os::LaunchNoncePipeDelivery> {
8670    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8671    // same maximum wait used for registration during a reload. Older modules
8672    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8673    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8674        Ok(delivery) => Some(delivery),
8675        Err(error) => {
8676            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8677            None
8678        }
8679    })
8680}
8681
8682fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8683    #[cfg(windows)]
8684    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8685        return Some(
8686            if snapshot
8687                .pipe_consumed
8688                .as_ref()
8689                .is_some_and(|consumed| consumed.get().is_some())
8690            {
8691                "pipe"
8692            } else {
8693                "env"
8694            }
8695            .to_owned(),
8696        );
8697    }
8698    let _ = snapshot;
8699    None
8700}
8701
8702/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8703/// of a blue/green swap.
8704#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8705enum SpawnRole {
8706    Plain,
8707    SwapCandidate,
8708}
8709
8710/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8711/// `apply_child_env` has already removed the variable for every spawn.
8712fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8713    if role == SpawnRole::SwapCandidate {
8714        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8715    }
8716}
8717
8718fn previous_instance_error(
8719    spec: &ModuleSpec,
8720    pid: u32,
8721    reason: crate::live_children::RetainedReason,
8722) -> SuperviseError {
8723    SuperviseError::Spawn {
8724        program: spec.program.clone(),
8725        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8726        cgroup_path: None,
8727    }
8728}
8729
8730fn spawn_child(
8731    spec: &ModuleSpec,
8732    connection_file_path: Option<&std::path::Path>,
8733    handle: Option<&SupervisorHandle>,
8734    ring: &Arc<Mutex<StderrRing>>,
8735    capture_logs_dir: Option<&std::path::Path>,
8736    roster: &ChildRoster,
8737    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8738) -> Result<SupervisedChild, SuperviseError> {
8739    spawn_child_in_slot(
8740        spec,
8741        connection_file_path,
8742        handle,
8743        ring,
8744        capture_logs_dir,
8745        roster,
8746        #[cfg(target_os = "linux")]
8747        cgroup_placement,
8748        SpawnRole::Plain,
8749        false,
8750    )
8751}
8752
8753/// Spawn one process of `spec` into a slot.
8754///
8755/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8756/// A swap candidate needs a different cgroup from the process it is replacing,
8757/// which is still alive: in the same cgroup the two would be one kill domain,
8758/// and killing a failed candidate could take the incumbent with it.
8759///
8760/// The stderr capture file is `<module_id>.stderr.log` for every process of
8761/// the module, whichever slot it is in, because that is the one file
8762/// `ck module logs` reads. During a swap's overlap both processes append to it;
8763/// the daemon writes whole lines, so the two interleave by line, which is also
8764/// the merged view an operator wants while a swap runs.
8765#[allow(clippy::too_many_arguments)]
8766fn spawn_child_in_slot(
8767    spec: &ModuleSpec,
8768    connection_file_path: Option<&std::path::Path>,
8769    handle: Option<&SupervisorHandle>,
8770    ring: &Arc<Mutex<StderrRing>>,
8771    capture_logs_dir: Option<&std::path::Path>,
8772    roster: &ChildRoster,
8773    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8774    role: SpawnRole,
8775    alternate_slot: bool,
8776) -> Result<SupervisedChild, SuperviseError> {
8777    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8778        return Err(previous_instance_error(spec, pid, reason));
8779    }
8780    if roster.is_closed() {
8781        return Err(SuperviseError::Spawn {
8782            program: spec.program.clone(),
8783            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8784            cgroup_path: None,
8785        });
8786    }
8787    #[cfg(target_os = "linux")]
8788    let cgroup_name = {
8789        // Slot names alone are not kill domains: a retired incumbent may still
8790        // be draining when a later enable/restart spawns into the same slot.
8791        // Decimal entropy keeps the suffix unambiguous; Placement performs
8792        // the module-id escaping and constructs the filesystem path.
8793        if cgroup_placement.is_none() {
8794            swap::cgroup_name(&spec.module_id, alternate_slot)
8795        } else {
8796            let nonce = generate_launch_nonce()?;
8797            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8798            // Leave room for byte escaping and the suffix under NAME_MAX. The
8799            // label is only for humans; the nonce identifies the kill domain.
8800            let mut end = spec.module_id.len().min(64);
8801            while !spec.module_id.is_char_boundary(end) {
8802                end -= 1;
8803            }
8804            format!(
8805                "{}_{suffix}",
8806                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8807            )
8808        }
8809    };
8810    #[cfg(not(target_os = "linux"))]
8811    let _ = alternate_slot;
8812    #[cfg(target_os = "macos")]
8813    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8814    #[cfg(windows)]
8815    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8816        program: spec.program.clone(),
8817        source,
8818        cgroup_path: None,
8819    })?;
8820    #[cfg(windows)]
8821    let mut command = Command::new(image_capture.path());
8822    #[cfg(not(any(target_os = "macos", windows)))]
8823    let mut command = Command::new(&spec.program);
8824    command.args(&spec.args);
8825    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8826    // that is the whole of the intent, so remove that one key rather than the
8827    // environment.
8828    //
8829    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8830    // and took the POSIX environment with it. Modules spawned that way had no
8831    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8832    // logging:
8833    //
8834    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8835    //     both unset it fell back to the temp dir alone and `ck` could not find
8836    //     a daemon running on the same machine from inside any module's process
8837    //     tree — reporting a path the file has never lived at, which reads as
8838    //     "the daemon did not write its file".
8839    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8840    //     the RELATIVE `.local/share`, so a module deriving its own store path
8841    //     resolved it against its own CWD. That is the store-fragmentation
8842    //     defect the daemon already refuses in config (`parse_doc` rejects a
8843    //     relative `storage.data_home`) arriving by derivation instead.
8844    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8845    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8846    //     quietly rather than erroring.
8847    //
8848    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8849    // offered one candidate under /tmp while the file sat in /run/user/1000.
8850    //
8851    // A configured module is unaffected either way: `module_spec()` puts the
8852    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8853    // wins over anything ambient.
8854    apply_child_env(&mut command, spec);
8855    apply_spawn_role(&mut command, role);
8856    let nonce_handoff =
8857        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8858
8859    #[cfg(target_os = "linux")]
8860    let cgroup_path = cgroup_placement
8861        .map(|placement| placement.module_path(&cgroup_name))
8862        .transpose()
8863        .map_err(|source| SuperviseError::Cgroup {
8864            module_id: spec.module_id.clone(),
8865            source,
8866        })?;
8867    #[cfg(not(target_os = "linux"))]
8868    let cgroup_path: Option<PathBuf> = None;
8869    #[cfg(target_os = "linux")]
8870    if let Some(path) = &cgroup_path {
8871        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8872            if let Some(placement) = cgroup_placement {
8873                remove_module_cgroup(placement, &cgroup_name);
8874            }
8875            return Err(error);
8876        }
8877    }
8878
8879    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8880        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8881        match ChildOutputSink::open(&path, capture_retention(spec)) {
8882            Ok(sink) => sink,
8883            Err(error) => {
8884                warn!(
8885                    module_id = %spec.module_id,
8886                    path = %path.display(),
8887                    error = %error,
8888                    "could not open child output capture file; forwarding to stderr"
8889                );
8890                ChildOutputSink::Stderr
8891            }
8892        }
8893    } else {
8894        ChildOutputSink::Stderr
8895    };
8896
8897    command.stdout(Stdio::piped());
8898    command.stderr(Stdio::piped());
8899    command.kill_on_drop(true);
8900    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8901    // before exec). In the daemon's group, a service manager that kills the
8902    // job's process group when the daemon exits (launchd's default) killed
8903    // every module at the same moment its control connection closed, so no
8904    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8905    // module is reached only by the daemon: the EOF it sees when its
8906    // connection closes, and the bounded stop in `child_roster` for anything
8907    // still running after that. On Linux this composes with the cgroup
8908    // placement above: that is a pre_exec write to cgroup.procs, std performs
8909    // setpgid in the child before running pre_exec callbacks, and the two
8910    // change independent process attributes.
8911    //
8912    // stdin is /dev/null because a process outside the terminal's foreground
8913    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8914    // by hand would otherwise hand down. Under a service manager stdin is
8915    // already /dev/null.
8916    #[cfg(unix)]
8917    command.process_group(0);
8918    command.stdin(Stdio::null());
8919    // The LAST pre-exec step, after the cgroup placement above: installing the
8920    // nonce at descriptor 3 replaces whatever the child had there, which could
8921    // be the descriptor an earlier step writes through.
8922    #[cfg(unix)]
8923    if let Some(handoff) = nonce_handoff {
8924        handoff.install_last(command.as_std_mut());
8925    }
8926    #[cfg(not(any(unix, windows)))]
8927    let _ = nonce_handoff;
8928
8929    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8930    // cannot run a single instruction -- and therefore cannot spawn a
8931    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8932    // other two steps and why the window matters.
8933    #[cfg(windows)]
8934    {
8935        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8936            program: spec.program.clone(),
8937            source,
8938            cgroup_path: None,
8939        })?;
8940        subc_jobobject::suspend_on_create_async(&mut command);
8941    }
8942    // Timestamp before the spawn call: on Unix the child can serve before that
8943    // call even returns. This conservative bound never labels an older child
8944    // as a replacement for a request received during process creation.
8945    let spawned_at = Instant::now();
8946    let mut child = match command.spawn() {
8947        Ok(child) => child,
8948        Err(source) => {
8949            #[cfg(target_os = "linux")]
8950            if let Some(placement) = cgroup_placement {
8951                remove_module_cgroup(placement, &cgroup_name);
8952            }
8953            return Err(SuperviseError::Spawn {
8954                program: spec.program.clone(),
8955                source,
8956                cgroup_path,
8957            });
8958        }
8959    };
8960    #[cfg(test)]
8961    spawn_record_tests::after_create(&child, spec);
8962    // The parent must close its writer now: the acknowledgement pipe reports EOF
8963    // only when every writer is gone, and the child's copy closes when the
8964    // trampoline replaces itself with the module. Command holds only an integer
8965    // in its pre_exec callback, not another writer.
8966    #[cfg(target_os = "macos")]
8967    drop(exec_ack);
8968
8969    // Bind while the child is still suspended, before job assignment resumes it.
8970    // Keeping the executable handle open across CreateProcessW denies rename
8971    // and deletion, so the file cannot be exchanged during spawn. Bind that
8972    // file ID to the process handle returned by spawn, never a reopened PID.
8973    #[cfg(windows)]
8974    let windows_image = match image_capture.bind_tokio(&child) {
8975        Ok(image) => Arc::new(image),
8976        Err(source) => {
8977            kill_and_reap_rejected_child(&mut child);
8978            return Err(SuperviseError::Spawn {
8979                program: spec.program.clone(),
8980                source,
8981                cgroup_path: None,
8982            });
8983        }
8984    };
8985    // Assign the suspended child to its module job; resume only after its
8986    // process identity has been durably recorded for orphan recovery.
8987    #[cfg(windows)]
8988    let job = contain_spawned_child(&mut child, spec)?;
8989    let spawned_at_ms = unix_ms_now();
8990    #[cfg(not(windows))]
8991    let spawned_from = spec.program.clone();
8992    #[cfg(windows)]
8993    let spawned_from = windows_image.path().to_path_buf();
8994    #[cfg(not(windows))]
8995    let spawned_file_identity = spawned_file_identity(&spawned_from);
8996    #[cfg(windows)]
8997    let spawned_file_identity = Some(SpawnedFileIdentity {
8998        device: 0,
8999        inode: 0,
9000        windows_image: Some(Arc::clone(&windows_image)),
9001    });
9002    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9003        program: spec.program.clone(),
9004        source: io::Error::other("spawned child exposed no live pid"),
9005        cgroup_path: cgroup_path.clone(),
9006    })?;
9007    #[cfg(windows)]
9008    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9009    #[cfg(not(windows))]
9010    let process_start_time = crate::provenance::process_start_time(pid);
9011    #[cfg(windows)]
9012    let process_start_time = Some(windows_image.start_time());
9013    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9014    #[cfg(all(test, target_os = "macos"))]
9015    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9016    // Record the committed kernel image, not the configured pathname: PATH
9017    // lookup and shebang interpretation can select a different executable.
9018    // Linux's posix_spawn can return before that image is installed, so the
9019    // observer also waits for the kernel's first-exec flag to clear.
9020    #[cfg(not(windows))]
9021    let recorded_image = observe_spawned_image(pid);
9022    // spawn() confirms only the first exec, into the trampoline. Never persist
9023    // the trampoline image; the asynchronous acknowledgement publishes the
9024    // module image once the trampoline has replaced itself with the module.
9025    #[cfg(target_os = "macos")]
9026    let recorded_image = if privacy_exec.is_some() {
9027        None
9028    } else {
9029        recorded_image
9030    };
9031    #[cfg(target_os = "linux")]
9032    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9033    #[cfg(not(target_os = "linux"))]
9034    let recorded_cgroup_name = None;
9035    let roster_guard = roster.admit(
9036        spec.module_id.clone(),
9037        pid,
9038        spec.protocol,
9039        process_start_time,
9040        crate::child_roster::RecordedIdentity {
9041            #[cfg(not(windows))]
9042            start_time: recorded_image.map(|image| image.start_time),
9043            #[cfg(windows)]
9044            start_time: process_start_time,
9045            #[cfg(not(windows))]
9046            executable: recorded_image
9047                .and_then(|image| image.executable)
9048                .map(crate::live_children::ExecutableIdentity::from),
9049            #[cfg(windows)]
9050            executable: Some(crate::live_children::ExecutableIdentity::from(
9051                windows_image.identity(),
9052            )),
9053            cgroup_name: recorded_cgroup_name,
9054            #[cfg(target_os = "linux")]
9055            cgroup_placement: cgroup_placement.cloned(),
9056        },
9057    );
9058    let roster_guard = match roster_guard {
9059        Ok(guard) => guard,
9060        Err(failure) => {
9061            return failure.with_cleanup(|source| {
9062                #[cfg(windows)]
9063                if let Some(job) = &job {
9064                    let _ = job.terminate();
9065                }
9066                #[cfg(target_os = "linux")]
9067                kill_module_cgroup(cgroup_placement, &cgroup_name);
9068                kill_and_reap_rejected_child(&mut child);
9069                #[cfg(target_os = "linux")]
9070                if let Some(placement) = cgroup_placement {
9071                    remove_module_cgroup(placement, &cgroup_name);
9072                }
9073                Err(SuperviseError::Spawn {
9074                    program: spec.program.clone(),
9075                    source: io::Error::other(format!(
9076                        "could not durably record module child: {source}"
9077                    )),
9078                    cgroup_path,
9079                })
9080            });
9081        }
9082    };
9083    // The pre-spawn check can pass just before shutdown begins. Ownership is
9084    // published at the start of admission, even while a record write is queued.
9085    // If shutdown already saw an empty roster before that publication, this
9086    // check sees the roster closed and ends the child before starting a module.
9087    if roster.is_closed() {
9088        // This child was never admitted, so there is no module protocol shutdown to wait for.
9089        #[cfg(target_os = "linux")]
9090        kill_module_cgroup(cgroup_placement, &cgroup_name);
9091        if let Err(error) = child.start_kill() {
9092            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9093        }
9094        #[cfg(target_os = "linux")]
9095        if let Some(placement) = cgroup_placement {
9096            // This spawn was never admitted, so shutdown has no roster entry
9097            // to await. Do not detach its cleanup: the runtime could exit
9098            // before that task reaps the rejected child and removes its group.
9099            while matches!(child.try_wait(), Ok(None)) {
9100                std::thread::yield_now();
9101            }
9102            if matches!(
9103                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9104                subc_cgroup::KillOutcome::Killed
9105            ) {
9106                if let Ok(path) = placement.module_path(&cgroup_name) {
9107                    while std::fs::read_to_string(path.join("cgroup.events"))
9108                        .ok()
9109                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9110                    {
9111                        std::thread::yield_now();
9112                    }
9113                }
9114            }
9115            remove_module_cgroup(placement, &cgroup_name);
9116        }
9117        drop(roster_guard);
9118        return Err(SuperviseError::Spawn {
9119            program: spec.program.clone(),
9120            source: io::Error::other(
9121                "the daemon began shutting down while this process was starting; ended it",
9122            ),
9123            cgroup_path,
9124        });
9125    }
9126
9127    #[cfg(windows)]
9128    if let Err(error) = resume_suspended_child(pid, spec) {
9129        kill_and_reap_rejected_child(&mut child);
9130        return Err(error);
9131    }
9132
9133    let stdout_pump = match child.stdout.take() {
9134        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9135        None => {
9136            warn!(
9137                module_id = %spec.module_id,
9138                "spawned child exposed no stdout pipe; file capture will be incomplete"
9139            );
9140            None
9141        }
9142    };
9143    let stderr_pump = match child.stderr.take() {
9144        Some(stderr) => {
9145            let generation = ring
9146                .lock()
9147                .unwrap_or_else(|poisoned| poisoned.into_inner())
9148                .begin_process();
9149            Some(StderrPump {
9150                task: tokio::spawn(pump_stderr_to(
9151                    stderr,
9152                    Arc::clone(ring),
9153                    generation,
9154                    output_sink,
9155                )),
9156                generation,
9157            })
9158        }
9159        None => {
9160            // Spawning succeeded but the pipe did not materialise. Recording it as
9161            // uncaptured keeps the tail honest: the alternative is an empty tail
9162            // that reads as a module which printed nothing.
9163            ring.lock()
9164                .unwrap_or_else(|poisoned| poisoned.into_inner())
9165                .mark_not_captured("stderr pipe was not available on spawn");
9166            warn!(
9167                module_id = %spec.module_id,
9168                "spawned child exposed no stderr pipe; tail will be unavailable"
9169            );
9170            None
9171        }
9172    };
9173
9174    let exec = Arc::new(crate::activity::ExecOutcome::new());
9175    #[cfg(target_os = "macos")]
9176    let pending_exec = privacy_exec.is_some();
9177    #[cfg(not(target_os = "macos"))]
9178    let pending_exec = false;
9179    if !pending_exec {
9180        exec.finish(crate::activity::ExecState::CommittedExec);
9181    }
9182    Ok(SupervisedChild {
9183        child,
9184        exec: crate::activity::ExecOwner(exec),
9185        #[cfg(windows)]
9186        nonce_delivery,
9187        protocol: spec.protocol,
9188        #[cfg(target_os = "linux")]
9189        module_id: cgroup_name,
9190        #[cfg(target_os = "linux")]
9191        cgroup_placement: cgroup_placement.cloned(),
9192        #[cfg(windows)]
9193        job,
9194        stdout_pump,
9195        stderr_pump,
9196        stderr_ring: Arc::clone(ring),
9197        spawned_at,
9198        registration_deadline: None,
9199        registration_watch: None,
9200        spawned_at_ms,
9201        spawned_from,
9202        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9203        spawned_file_identity,
9204        process_start_time,
9205        process_identity,
9206        pid,
9207        roster_guard: Some(roster_guard),
9208        #[cfg(target_os = "macos")]
9209        privacy_exec,
9210        #[cfg(target_os = "macos")]
9211        report_ready: Arc::new(OnceLock::new()),
9212        spawn_failure: None,
9213    })
9214}
9215
9216#[cfg(test)]
9217mod spawn_record_tests {
9218    use super::*;
9219
9220    thread_local! {
9221        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9222    }
9223
9224    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9225        LAST_PID.set(child.id());
9226        #[cfg(windows)]
9227        if let Some((_, address)) = _spec
9228            .env
9229            .iter()
9230            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9231        {
9232            use std::io::{Read, Write};
9233            let mut stream = std::net::TcpStream::connect(address).unwrap();
9234            stream
9235                .write_all(&child.id().unwrap().to_ne_bytes())
9236                .unwrap();
9237            let mut release = [0];
9238            stream.read_exact(&mut release).unwrap();
9239        }
9240    }
9241
9242    #[cfg(windows)]
9243    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9244        if spec
9245            .env
9246            .iter()
9247            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9248        {
9249            Err(io::Error::other("injected module job assignment failure"))
9250        } else {
9251            Ok(())
9252        }
9253    }
9254
9255    fn spec() -> ModuleSpec {
9256        ModuleSpec {
9257            module_id: "spawn-record-worker".into(),
9258            #[cfg(unix)]
9259            program: "/bin/sleep".into(),
9260            #[cfg(windows)]
9261            program: std::env::current_exe().unwrap(),
9262            #[cfg(unix)]
9263            args: vec!["60".into()],
9264            #[cfg(windows)]
9265            args: vec![
9266                "--exact".into(),
9267                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9268                "--nocapture".into(),
9269            ],
9270            env: vec![],
9271            reserved: false,
9272            reserved_prefixes: vec![],
9273            protocol: ModuleProtocol::None,
9274            overlap: Default::default(),
9275        }
9276    }
9277
9278    #[tokio::test]
9279    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9280        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9281        let supervisor =
9282            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9283                .with_live_children_record(home.join("live-children.json"));
9284        crate::live_children::fail_next_file_sync();
9285        let result = supervisor.spawn(spec());
9286        let failed = result.is_err();
9287        if let Ok(module) = result {
9288            module.stop().await.unwrap();
9289        }
9290        assert!(failed, "record failure must fail spawn");
9291        let pid = LAST_PID.get().expect("the real OS child was created");
9292        assert!(
9293            subc_os::Process::open(pid).unwrap().is_none(),
9294            "child {pid} survived rejected admission or was not reaped"
9295        );
9296    }
9297
9298    #[cfg(target_os = "macos")]
9299    #[test]
9300    #[ignore = "manual on-device latency measurement"]
9301    fn durable_record_write_latency_50() {
9302        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9303        let path = home.join("live-children.json");
9304        let children = vec![crate::live_children::LiveChild {
9305            module_id: "latency-worker".into(),
9306            pid: std::process::id(),
9307            protocol: ModuleProtocol::None,
9308            start_time: Some(1),
9309            executable: None,
9310            cgroup_name: None,
9311        }];
9312        let mut samples = Vec::new();
9313        for _ in 0..50 {
9314            let start = std::time::Instant::now();
9315            crate::live_children::write_record(&path, &children).unwrap();
9316            samples.push(start.elapsed());
9317        }
9318        samples.sort();
9319        eprintln!(
9320            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9321            (samples[24] + samples[25]) / 2,
9322            samples[47]
9323        );
9324    }
9325
9326    #[cfg(windows)]
9327    #[test]
9328    fn windows_store_touch_fixture() {
9329        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9330            std::fs::write(path, b"module ran").unwrap();
9331        }
9332    }
9333
9334    #[cfg(windows)]
9335    #[tokio::test]
9336    async fn windows_daemon_fixture() {
9337        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9338            return;
9339        };
9340        let mut spec = spec();
9341        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9342        let supervisor =
9343            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9344        let _module = supervisor.spawn(spec).unwrap();
9345    }
9346
9347    #[cfg(windows)]
9348    #[test]
9349    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9350        use std::io::Read;
9351        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9352        // An outer job models a scheduler/service host. The daemon's root and
9353        // root job must nest without requiring breakaway from this job.
9354        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9355        command
9356            .args([
9357                "--exact",
9358                "supervise::spawn_record_tests::windows_daemon_fixture",
9359                "--nocapture",
9360            ])
9361            .env(
9362                "SUBC_TEST_DAEMON_BARRIER",
9363                listener.local_addr().unwrap().to_string(),
9364            );
9365        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9366        listener.set_nonblocking(true).unwrap();
9367        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9368        let mut stream = loop {
9369            if let Ok((stream, _)) = listener.accept() {
9370                break stream;
9371            }
9372            assert!(
9373                daemon.child.try_wait().unwrap().is_none(),
9374                "daemon failed before rendezvous"
9375            );
9376            assert!(
9377                std::time::Instant::now() < deadline,
9378                "daemon did not reach CreateProcess failpoint"
9379            );
9380            std::thread::yield_now();
9381        };
9382        stream
9383            .set_read_timeout(Some(Duration::from_secs(30)))
9384            .unwrap();
9385        let mut pid = [0; 4];
9386        stream.read_exact(&mut pid).unwrap();
9387        let pid = u32::from_ne_bytes(pid);
9388        assert!(subc_jobobject::process_exists(pid));
9389        // Terminate only the daemon, not its outer job: otherwise the host job
9390        // would also kill a child whose daemon never joined its root job.
9391        daemon.child.kill().unwrap();
9392        daemon.child.wait().unwrap();
9393        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9394        if !exited {
9395            daemon.job.terminate().unwrap();
9396        }
9397        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9398    }
9399
9400    #[cfg(windows)]
9401    #[tokio::test]
9402    async fn windows_containment_failure_never_resumes() {
9403        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9404        let sentinel = home.join("store-touched");
9405        let mut spec = spec();
9406        spec.env
9407            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9408        spec.env.push((
9409            "SUBC_TEST_STORE_TOUCH".into(),
9410            sentinel.display().to_string(),
9411        ));
9412        let supervisor =
9413            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9414        let result = supervisor.spawn(spec);
9415        // A version that resumes despite assignment failure must have time to
9416        // execute its sentinel before dropping the returned module kills it.
9417        if let Ok(module) = result {
9418            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9419                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9420                    tokio::task::yield_now().await;
9421                }
9422            })
9423            .await;
9424            let _ = module.stop().await;
9425            panic!(
9426                "assignment failure resumed a module; store touched={}",
9427                sentinel.exists()
9428            );
9429        }
9430        assert!(
9431            !sentinel.exists(),
9432            "refused module executed store-touch sentinel"
9433        );
9434        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9435    }
9436
9437    #[cfg(windows)]
9438    #[tokio::test]
9439    async fn windows_nested_daemon_fixture() {
9440        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9441            return;
9442        };
9443        let mut spec = spec();
9444        spec.env.push((
9445            "SUBC_TEST_STORE_TOUCH".into(),
9446            sentinel.to_str().unwrap().into(),
9447        ));
9448        let supervisor =
9449            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9450        let module = supervisor
9451            .spawn(spec)
9452            .expect("per-module job must nest below daemon and host jobs");
9453        tokio::time::timeout(Duration::from_secs(10), async {
9454            while !PathBuf::from(&sentinel).exists() {
9455                tokio::task::yield_now().await;
9456            }
9457        })
9458        .await
9459        .unwrap();
9460        module.stop().await.unwrap();
9461    }
9462
9463    #[cfg(windows)]
9464    #[test]
9465    fn windows_module_job_nests_under_host_job() {
9466        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9467        let sentinel = home.join("nested-store-touched");
9468        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9469        command
9470            .args([
9471                "--exact",
9472                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9473                "--nocapture",
9474            ])
9475            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9476        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9477        assert!(daemon.child.wait().unwrap().success());
9478        assert!(sentinel.exists(), "nested protocol:none module never ran");
9479    }
9480}
9481
9482#[cfg(target_os = "linux")]
9483pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9484    use subc_cgroup::KillOutcome;
9485    match subc_cgroup::kill_module(placement, module_id) {
9486        KillOutcome::Killed => {}
9487        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9488            debug!(
9489                module_id,
9490                "cgroup tree kill unavailable; using direct-child kill"
9491            );
9492        }
9493        KillOutcome::IoError { path, error } => {
9494            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9495        }
9496    }
9497}
9498
9499/// Assign a freshly spawned Windows child without ever resuming on failure.
9500///
9501/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9502/// child assigned **while it is still suspended** (step 1 is
9503/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9504/// the child's roster admission is durable. Failed containment kills and reaps
9505/// the suspended child instead of running an unprotected module.
9506#[cfg(windows)]
9507fn contain_spawned_child(
9508    child: &mut Child,
9509    spec: &ModuleSpec,
9510) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9511    let assignment = || {
9512        let job = subc_jobobject::JobObject::new()?;
9513        #[cfg(test)]
9514        spawn_record_tests::assignment_failpoint(spec)?;
9515        job.assign(child)?;
9516        Ok::<_, io::Error>(job)
9517    };
9518    match assignment() {
9519        Ok(job) => Ok(Some(job)),
9520        Err(source) => {
9521            kill_and_reap_rejected_child(child);
9522            Err(SuperviseError::Spawn {
9523                program: spec.program.clone(),
9524                source: io::Error::other(format!(
9525                    "Windows module job containment failed: {source}"
9526                )),
9527                cgroup_path: None,
9528            })
9529        }
9530    }
9531}
9532
9533/// Rejection is synchronous: do not return a spawn error while the child can
9534/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9535fn kill_and_reap_rejected_child(child: &mut Child) {
9536    #[cfg(unix)]
9537    if let Some(pid) = child
9538        .id()
9539        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9540    {
9541        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9542    }
9543    let _ = child.start_kill();
9544    while matches!(child.try_wait(), Ok(None)) {
9545        std::thread::yield_now();
9546    }
9547}
9548
9549/// Resume a suspended child, killing it if it cannot be started.
9550///
9551/// A suspended process holds a pid and does nothing, so there is no useful
9552/// state to return: the caller gets an error and the spawn fails.
9553#[cfg(windows)]
9554fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9555    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9556        // Kill it here rather than leaving a suspended process for the caller
9557        // to notice; `kill_on_drop` would eventually do this, but the module
9558        // would have been reported as running in between.
9559        let _ = std::process::Command::new("taskkill.exe")
9560            .args(["/PID", &pid.to_string(), "/T", "/F"])
9561            .stdin(Stdio::null())
9562            .stdout(Stdio::null())
9563            .stderr(Stdio::null())
9564            .status();
9565        return Err(SuperviseError::Spawn {
9566            program: spec.program.clone(),
9567            source,
9568            cgroup_path: None,
9569        });
9570    }
9571    Ok(())
9572}
9573
9574#[cfg(target_os = "linux")]
9575fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9576    match placement.remove_module(module_id) {
9577        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9578        Err(error) => warn!(
9579            module_id,
9580            error = %error,
9581            "could not remove module cgroup after process exit; continuing teardown"
9582        ),
9583    }
9584}
9585
9586#[cfg(target_os = "linux")]
9587const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
9588#[cfg(target_os = "linux")]
9589const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
9590
9591#[cfg(target_os = "linux")]
9592async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9593    let path = placement.module_path(module_id).ok();
9594    if path.as_ref().is_some_and(|path| {
9595        std::fs::read_to_string(path.join("cgroup.events"))
9596            .ok()
9597            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
9598    }) {
9599        remove_module_cgroup(placement, module_id);
9600        return;
9601    }
9602
9603    // Freeze before enumerating: a frozen task cannot exit and release its PID
9604    // for reuse by a process outside this module while SIGTERM is being sent.
9605    if terminate_reaped_cgroup(placement, module_id).await {
9606        if let Some(path) = &path {
9607            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
9608            while cgroup_is_populated(path) && Instant::now() < deadline {
9609                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
9610            }
9611        }
9612    }
9613
9614    // Reaping the direct child is not proof its descendants exited. Kill any
9615    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
9616    // cgroup empty before rmdir.
9617    if matches!(
9618        subc_cgroup::kill_module(Some(placement), module_id),
9619        subc_cgroup::KillOutcome::Killed
9620    ) {
9621        if let Some(path) = &path {
9622            while cgroup_is_populated(path) {
9623                sleep(Duration::from_millis(1)).await;
9624            }
9625        }
9626    }
9627    remove_module_cgroup(placement, module_id);
9628}
9629
9630#[cfg(target_os = "linux")]
9631fn cgroup_is_populated(path: &std::path::Path) -> bool {
9632    std::fs::read_to_string(path.join("cgroup.events"))
9633        .ok()
9634        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9635}
9636
9637#[cfg(target_os = "linux")]
9638async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
9639    let frozen =
9640        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
9641            .await
9642        {
9643            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
9644            outcome => {
9645                tracing::debug!(
9646                    module_id,
9647                    ?outcome,
9648                    "cgroup freeze failed; skipping SIGTERM grace"
9649                );
9650                return false;
9651            }
9652        };
9653    let listed = match frozen.processes() {
9654        subc_cgroup::ProcsOutcome::Listed(pids) => {
9655            for pid in pids.into_iter().filter(|pid| *pid > 1) {
9656                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
9657                    if let Err(error) =
9658                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
9659                    {
9660                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
9661                    }
9662                }
9663            }
9664            true
9665        }
9666        outcome => {
9667            tracing::debug!(
9668                module_id,
9669                ?outcome,
9670                "cgroup process listing failed; skipping SIGTERM grace"
9671            );
9672            false
9673        }
9674    };
9675    match frozen.thaw() {
9676        subc_cgroup::ThawOutcome::Thawed => listed,
9677        outcome => {
9678            tracing::debug!(
9679                module_id,
9680                ?outcome,
9681                "cgroup thaw failed; proceeding to cgroup.kill"
9682            );
9683            false
9684        }
9685    }
9686}
9687
9688#[cfg(target_os = "linux")]
9689fn apply_cgroup_placement(
9690    command: &mut Command,
9691    spec: &ModuleSpec,
9692    path: &std::path::Path,
9693) -> Result<(), SuperviseError> {
9694    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9695        module_id: spec.module_id.clone(),
9696        source,
9697    })
9698}
9699
9700fn capture_retention(spec: &ModuleSpec) -> Retention {
9701    let defaults = Retention::default();
9702    let value = |name: &str| {
9703        spec.env
9704            .iter()
9705            .rev()
9706            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9707    };
9708    Retention {
9709        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9710            .and_then(|value| value.parse().ok())
9711            .unwrap_or(defaults.max_file_mb),
9712        keep: value(CAPTURE_KEEP_ENV)
9713            .and_then(|value| value.parse().ok())
9714            .unwrap_or(defaults.keep),
9715        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9716            .and_then(|value| value.parse().ok())
9717            .unwrap_or(defaults.max_age_days),
9718    }
9719}
9720
9721/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9722/// module's registration to the exact process the supervisor spawned.
9723fn generate_launch_nonce() -> Result<String, SuperviseError> {
9724    let mut bytes = [0u8; 32];
9725    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9726        reason: source.to_string(),
9727    })?;
9728    let mut hex = String::with_capacity(64);
9729    for b in bytes {
9730        use std::fmt::Write;
9731        let _ = write!(hex, "{b:02x}");
9732    }
9733    Ok(hex)
9734}
9735
9736/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9737/// signal about how many leading bytes matched.
9738fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9739    if a.len() != b.len() {
9740        return false;
9741    }
9742    let mut diff = 0u8;
9743    for (x, y) in a.iter().zip(b.iter()) {
9744        diff |= x ^ y;
9745    }
9746    diff == 0
9747}
9748
9749/// The kernel's image after an acknowledged exec, shared by ordinary launches
9750/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9751/// not identities inferred from a configured pathname.
9752#[cfg(not(windows))]
9753fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9754    let process = subc_os::Process::open(pid).ok().flatten()?;
9755    #[cfg(target_os = "linux")]
9756    return sample_after_linux_exec(
9757        || {
9758            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9759            let committed = linux_exec_committed(&stat);
9760            #[cfg(test)]
9761            spawn_image_tests::report_exec_state(committed);
9762            committed
9763        },
9764        || process.observe(),
9765    );
9766    #[cfg(not(target_os = "linux"))]
9767    process.observe()
9768}
9769
9770/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9771/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9772/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9773#[cfg(target_os = "linux")]
9774fn linux_exec_committed(stat: &str) -> Option<bool> {
9775    const PF_FORKNOEXEC: u64 = 0x40;
9776    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9777    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9778}
9779
9780#[cfg(target_os = "linux")]
9781fn sample_after_linux_exec(
9782    mut exec_committed: impl FnMut() -> Option<bool>,
9783    observe: impl FnOnce() -> Option<subc_os::Observation>,
9784) -> Option<subc_os::Observation> {
9785    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9786    while !exec_committed()? {
9787        // A failed or indefinitely stalled exec must not publish the inherited
9788        // image as authority to signal a future orphan. Missing identity keeps
9789        // the existing conservative orphan-sweep behavior.
9790        if std::time::Instant::now() >= deadline {
9791            return None;
9792        }
9793        std::thread::sleep(Duration::from_micros(50));
9794    }
9795    observe()
9796}
9797
9798#[cfg(all(test, target_os = "linux"))]
9799mod spawn_image_tests {
9800    use super::*;
9801    use std::cell::{Cell, RefCell};
9802
9803    enum ExecEvent {
9804        Uncommitted,
9805        Image(Option<subc_os::Observation>),
9806    }
9807
9808    thread_local! {
9809        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9810    }
9811
9812    pub(super) fn report_exec_state(committed: Option<bool>) {
9813        if committed == Some(false) {
9814            EXEC_EVENTS.with(|events| {
9815                if let Some(events) = events.borrow_mut().take() {
9816                    let _ = events.send(ExecEvent::Uncommitted);
9817                }
9818            });
9819        }
9820    }
9821
9822    #[test]
9823    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9824        use std::io::{Read, Write};
9825        use std::os::fd::AsRawFd;
9826
9827        let descriptor = std::fs::File::open("/dev/null").unwrap();
9828        let mut command = std::process::Command::new("/bin/sleep");
9829        command.arg("30");
9830        let (mut ready, mut release) =
9831            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9832                .unwrap();
9833        let spawn = std::thread::spawn(move || command.spawn());
9834        let mut message = [0u8; 8];
9835        ready.read_exact(&mut message).unwrap();
9836        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9837        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9838        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9839        let (events, received) = std::sync::mpsc::channel();
9840        let observer = std::thread::spawn(move || {
9841            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9842            let image = observe_spawned_image(pid);
9843            let _ = events.send(ExecEvent::Image(image));
9844        });
9845        // Release only once the production observer has seen the uncommitted
9846        // exec, or has incorrectly returned the inherited image. Either way,
9847        // release and reap before asserting so a regression cannot leak a child.
9848        let first = received.recv_timeout(Duration::from_secs(5));
9849        release.write_all(&[1]).unwrap();
9850        let mut child = spawn.join().unwrap().unwrap();
9851        let after = match first {
9852            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9853                Ok(ExecEvent::Image(image)) => image,
9854                _ => None,
9855            },
9856            Ok(ExecEvent::Image(image)) => image,
9857            Err(_) => None,
9858        };
9859        observer.join().unwrap();
9860        child.kill().unwrap();
9861        child.wait().unwrap();
9862
9863        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9864        assert_eq!(
9865            inherited.unwrap().executable,
9866            subc_os::file_identity(&std::env::current_exe().unwrap())
9867        );
9868        assert_eq!(
9869            after.unwrap().executable,
9870            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9871        );
9872    }
9873
9874    #[test]
9875    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9876        let committed = Cell::new(false);
9877        let inherited = subc_os::Observation {
9878            start_time: 23,
9879            executable: Some(subc_os::FileIdentity {
9880                device: 29,
9881                inode: 31,
9882            }),
9883        };
9884        let module = subc_os::Observation {
9885            start_time: 23,
9886            executable: Some(subc_os::FileIdentity {
9887                device: 29,
9888                inode: 37,
9889            }),
9890        };
9891        let polls = Cell::new(0);
9892        let image = sample_after_linux_exec(
9893            || {
9894                polls.set(polls.get() + 1);
9895                // Hold the inherited image through the first poll, as in the
9896                // posix_spawn parent-wake window, then commit the module image.
9897                committed.set(polls.get() == 2);
9898                Some(committed.get())
9899            },
9900            || Some(if committed.get() { module } else { inherited }),
9901        );
9902        assert_eq!(image, Some(module));
9903        assert_eq!(polls.get(), 2);
9904    }
9905
9906    #[test]
9907    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9908        assert_eq!(
9909            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9910            Some(false)
9911        );
9912        assert_eq!(
9913            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9914            Some(true)
9915        );
9916        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9917        assert_eq!(linux_exec_committed("17 (a) R"), None);
9918    }
9919
9920    #[test]
9921    fn unreadable_exec_state_never_samples_an_inherited_image() {
9922        assert_eq!(
9923            sample_after_linux_exec(
9924                || None,
9925                || panic!("must not sample before exec confirmation")
9926            ),
9927            None
9928        );
9929    }
9930
9931    #[test]
9932    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9933        assert_eq!(
9934            sample_after_linux_exec(
9935                || Some(false),
9936                || panic!("must not sample after an exec timeout")
9937            ),
9938            None
9939        );
9940    }
9941}
9942
9943/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9944/// different executable after the no-delete capture was taken.
9945#[cfg(windows)]
9946fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9947    let mut program = spec.program.clone();
9948    if program.is_absolute() || program.components().count() > 1 {
9949        // An explicit path can name a PE image with no extension. Try that
9950        // exact file first, as Command does, before falling back to .exe.
9951        match subc_os::ExecutableCapture::open(&program) {
9952            Ok(capture) => return Ok(capture),
9953            Err(error)
9954                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9955            Err(error) => return Err(error),
9956        }
9957        program.set_extension("exe");
9958        return subc_os::ExecutableCapture::open(&program);
9959    }
9960    if program.extension().is_none() {
9961        program.set_extension("exe");
9962    }
9963    let custom_path = spec
9964        .env
9965        .iter()
9966        .rev()
9967        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9968    let mut directories = Vec::new();
9969    // These are Windows' default executable search locations. A configured
9970    // PATH still overrides the inherited PATH, not the system directories.
9971    if let Some(parent) = std::env::current_exe()?.parent() {
9972        directories.push(parent.to_path_buf());
9973    }
9974    directories.push(std::env::current_dir()?);
9975    if let Some(windows) = std::env::var_os("SystemRoot") {
9976        let windows = PathBuf::from(windows);
9977        directories.push(windows.join("System32"));
9978        directories.push(windows.clone());
9979    }
9980    let path = custom_path
9981        .map(|(_, value)| std::ffi::OsString::from(value))
9982        .or_else(|| std::env::var_os("PATH"));
9983    if let Some(path) = path {
9984        directories.extend(std::env::split_paths(&path));
9985    }
9986    for directory in directories {
9987        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9988            Ok(capture) => return Ok(capture),
9989            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9990            Err(error) => return Err(error),
9991        }
9992    }
9993    Err(io::Error::new(
9994        io::ErrorKind::NotFound,
9995        "resolved executable not found",
9996    ))
9997}
9998
9999fn spawn_and_mark_running(
10000    spec: &ModuleSpec,
10001    runtime: &SupervisorRuntimeConfig,
10002    snapshot: &SharedSnapshot,
10003) -> Result<SupervisedChild, SuperviseError> {
10004    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10005    let registration_watch = watch_registration(&runtime.registry, spec);
10006    let child = spawn_child(
10007        spec,
10008        runtime.connection_file_path.as_deref(),
10009        runtime.supervisor_handle.as_ref(),
10010        &runtime.stderr_ring,
10011        runtime.capture_logs_dir.as_deref(),
10012        &runtime.child_roster,
10013        #[cfg(target_os = "linux")]
10014        runtime.cgroup_placement.as_ref(),
10015    )?;
10016    let mut child = child;
10017    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10018    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
10019    if let Some(spawn) = activity_spawn {
10020        spawn.complete(snapshot);
10021    }
10022    Ok(child)
10023}
10024
10025struct RegistrationWatch {
10026    changes: watch::Receiver<u64>,
10027    baseline: u64,
10028}
10029
10030// Capture before spawning: a fast child's accepted HELLO can disappear from the
10031// active registry before the monitor first runs. A pre-existing registration
10032// must not satisfy a new process's deadline either.
10033fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10034    if spec.protocol != ModuleProtocol::Subc {
10035        return None;
10036    }
10037    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10038    let baseline = *changes.borrow();
10039    Some(RegistrationWatch { changes, baseline })
10040}
10041
10042fn arm_registration_deadline(
10043    child: &mut SupervisedChild,
10044    budget: Duration,
10045    registration_watch: Option<RegistrationWatch>,
10046) {
10047    child.registration_deadline =
10048        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10049    child.registration_watch = registration_watch;
10050}
10051
10052async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10053    #[cfg(unix)]
10054    {
10055        // Children lead their own process groups. Signal the whole group so a
10056        // helper stuck before HELLO cannot survive its supervised parent.
10057        let group = unreaped_registration_group(child)?;
10058        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10059        // Process-group safety rule: only signal a group while its leader is
10060        // our unreaped child. Even a zombie leader reserves its pid/group id;
10061        // reaping it before KILL would let that id name an unrelated group.
10062        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10063        sleep(crate::child_roster::TERM_TO_KILL).await;
10064        let group = unreaped_registration_group(child)?;
10065        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10066    }
10067    // Windows terminates the containing Job Object as well as the direct
10068    // child. Linux also kills the child's cgroup when placement is enabled.
10069    child.start_kill()?;
10070    child.wait().await?;
10071    Ok(())
10072}
10073
10074#[cfg(unix)]
10075fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10076    // The group id comes only from the pid recorded at spawn, when the child
10077    // created its own group. Never admit init's group or a negative group id.
10078    let group = i32::try_from(child.pid)
10079        .ok()
10080        .filter(|pid| *pid > 1)
10081        .and_then(rustix::process::Pid::from_raw)
10082        .ok_or_else(|| {
10083            io::Error::new(
10084                io::ErrorKind::InvalidInput,
10085                "invalid supervised process group",
10086            )
10087        })?;
10088    if child.child.id() != Some(child.pid) {
10089        return Err(io::Error::new(
10090            io::ErrorKind::InvalidInput,
10091            "refusing process-group signal after supervised child was reaped",
10092        ));
10093    }
10094    Ok(group)
10095}
10096
10097enum RegistrationWaitOutcome {
10098    Registered,
10099    Exited(ExitReport),
10100    TimedOut,
10101}
10102
10103struct ReloadRegistrationFailure {
10104    exit_report: ExitReport,
10105    reason: String,
10106}
10107
10108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10109enum BusyGaugeObservation {
10110    Quiescent,
10111    Busy,
10112    Omitted,
10113}
10114
10115/// What one health answer said about one declared busy gauge.
10116#[derive(Debug, Clone, PartialEq, Eq)]
10117enum GaugeVerdict {
10118    /// The gauge read 0: this gauge is not what holds the drain.
10119    Zero,
10120    /// The gauge read a positive count.
10121    NonZero(u64),
10122    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10123    /// a negative number). It counts as busy, because the daemon cannot tell
10124    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10125    NotACount(String),
10126    /// The health answer had no metrics object, or one without this gauge.
10127    Missing,
10128    /// The probe produced no health answer to read; carries the probe error's
10129    /// evidence label (`no-answer` for a probe that timed out).
10130    ProbeFailed(&'static str),
10131}
10132
10133impl GaugeVerdict {
10134    fn reads_quiet(&self) -> bool {
10135        matches!(self, Self::Zero)
10136    }
10137}
10138
10139impl fmt::Display for GaugeVerdict {
10140    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10141        match self {
10142            Self::Zero => f.write_str("zero"),
10143            Self::NonZero(value) => write!(f, "non_zero({value})"),
10144            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10145            Self::Missing => f.write_str("missing_from_health_body"),
10146            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10147        }
10148    }
10149}
10150
10151/// The longest JSON text kept for a gauge value that is not a count, so a
10152/// module reporting a large object as a gauge cannot flood the drain log.
10153const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10154
10155/// Read each declared gauge out of one health answer's metrics.
10156fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10157    let metrics = metrics.and_then(Value::as_object);
10158    gauges
10159        .iter()
10160        .map(
10161            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10162                None => GaugeVerdict::Missing,
10163                Some(value) => match value.as_u64() {
10164                    Some(0) => GaugeVerdict::Zero,
10165                    Some(count) => GaugeVerdict::NonZero(count),
10166                    None => {
10167                        let mut text = value.to_string();
10168                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10169                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10170                            while !text.is_char_boundary(end) {
10171                                end -= 1;
10172                            }
10173                            text.truncate(end);
10174                            text.push_str("...");
10175                        }
10176                        GaugeVerdict::NotACount(text)
10177                    }
10178                },
10179            },
10180        )
10181        .collect()
10182}
10183
10184/// The drain's overall reading of one health answer: omitted when any
10185/// declared gauge is missing, quiescent only when every gauge reads 0.
10186fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10187    if verdicts
10188        .iter()
10189        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10190    {
10191        BusyGaugeObservation::Omitted
10192    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10193        BusyGaugeObservation::Quiescent
10194    } else {
10195        BusyGaugeObservation::Busy
10196    }
10197}
10198
10199/// One declared gauge's history over a drain: the latest verdict on it, and
10200/// the latest value actually read for it (which a later failed probe or
10201/// missing gauge does not erase). Times are offsets from the start of the
10202/// drain's wait.
10203#[derive(Debug, Clone, Default, PartialEq, Eq)]
10204struct GaugeTrace {
10205    latest: Option<(GaugeVerdict, Duration)>,
10206    last_value: Option<(GaugeVerdict, Duration)>,
10207}
10208
10209/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10210/// its ceiling can say in its timeout line which declared gauge held it and
10211/// why. Without it, a gauge that read busy, a gauge the module never
10212/// reported, and a probe that never got an answer all looked the same: a
10213/// drain that ended at its ceiling with nothing in flight.
10214#[derive(Debug, Clone, Default, PartialEq, Eq)]
10215struct DrainGaugeTrace {
10216    /// Health probes the drain sent.
10217    probes: u32,
10218    /// Each declared gauge, in declaration order.
10219    gauges: Vec<(String, GaugeTrace)>,
10220}
10221
10222impl DrainGaugeTrace {
10223    fn new(gauges: &[String]) -> Self {
10224        Self {
10225            probes: 0,
10226            gauges: gauges
10227                .iter()
10228                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10229                .collect(),
10230        }
10231    }
10232
10233    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10234    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10235        self.probes = self.probes.saturating_add(1);
10236        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10237            trace.latest = Some((verdict.clone(), at));
10238            if matches!(
10239                verdict,
10240                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10241            ) {
10242                trace.last_value = Some((verdict.clone(), at));
10243            }
10244        }
10245    }
10246}
10247
10248/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10249/// declared gauge, separated by ` | `. A value is shown as its verdict
10250/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10251/// and whether that counted as busy.
10252impl fmt::Display for DrainGaugeTrace {
10253    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10254        if self.gauges.is_empty() {
10255            return f.write_str("none declared");
10256        }
10257        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10258            if index > 0 {
10259                f.write_str(" | ")?;
10260            }
10261            write!(f, "{name}: ")?;
10262            match &trace.latest {
10263                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10264                None => f.write_str("never probed")?,
10265            }
10266            match &trace.last_value {
10267                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10268                None => f.write_str("; no value ever read")?,
10269            }
10270        }
10271        Ok(())
10272    }
10273}
10274
10275pub(crate) fn declared_busy_gauges(
10276    registry: &Registry,
10277    module_id: &str,
10278) -> Result<Vec<String>, SuperviseError> {
10279    busy_gauges_of(
10280        registry
10281            .get_module(module_id)
10282            .map_err(SuperviseError::Registry)?,
10283    )
10284}
10285
10286/// [`declared_busy_gauges`] for the registration a connection holds, in any
10287/// slot: after cutover the incumbent is no longer the id's active
10288/// registration, and its own manifest is the one that names its gauges.
10289fn declared_busy_gauges_for_connection(
10290    registry: &Registry,
10291    connection_id: ConnectionId,
10292) -> Result<Vec<String>, SuperviseError> {
10293    busy_gauges_of(
10294        registry
10295            .get_module_by_connection(connection_id)
10296            .map_err(SuperviseError::Registry)?,
10297    )
10298}
10299
10300fn busy_gauges_of(
10301    registration: Option<crate::registry::ModuleRegistration>,
10302) -> Result<Vec<String>, SuperviseError> {
10303    let Some(registration) = registration else {
10304        return Ok(Vec::new());
10305    };
10306    let Some(self_signals) = registration.manifest.self_signals else {
10307        return Ok(Vec::new());
10308    };
10309
10310    let mut gauges = Vec::new();
10311    for declaration in self_signals {
10312        if declaration.kind != SelfSignalKind::Busy {
10313            continue;
10314        }
10315        match declaration.anchored_to {
10316            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10317                gauges.extend(declared)
10318            }
10319            _ => {
10320                // An invalid Busy anchor is fail-safe: the empty name cannot be
10321                // present in a conforming health report, so this drain stays busy.
10322                gauges.push(String::new());
10323            }
10324        }
10325    }
10326    Ok(gauges)
10327}
10328
10329/// Wait for `endpoint` to have nothing in flight and, when the module declares
10330/// busy gauges, for a health probe to report them quiet. The probe is addressed
10331/// by `scope`: a swap's superseded incumbent must be asked about its own
10332/// gauges, and by module id the probe would reach the promoted candidate.
10333/// While the gauges read busy, the probe repeats every
10334/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10335/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10336/// timeout line.
10337#[allow(clippy::too_many_arguments)]
10338async fn wait_for_forwarding_quiescence(
10339    forwarding: &ForwardingTable,
10340    module_id: &str,
10341    runtime: &SupervisorRuntimeConfig,
10342    endpoint: crate::ModuleEndpointId,
10343    deadline: Instant,
10344    busy_gauges: &[String],
10345    scope: DrainScope,
10346    trace: &mut DrainGaugeTrace,
10347) -> Result<bool, SuperviseError> {
10348    let wait_started = Instant::now();
10349    let mut gauges_quiescent = busy_gauges.is_empty();
10350    let mut next_probe_at = wait_started;
10351    let mut omission_reported = false;
10352
10353    loop {
10354        let now = Instant::now();
10355        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10356            let report = match scope {
10357                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10358                DrainScope::Endpoint(endpoint) => {
10359                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10360                }
10361            };
10362            let verdicts = match &report {
10363                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10364                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10365            };
10366            trace.record(&verdicts, wait_started.elapsed());
10367            gauges_quiescent = match report {
10368                Ok(_) => match busy_gauge_observation(&verdicts) {
10369                    BusyGaugeObservation::Quiescent => true,
10370                    BusyGaugeObservation::Busy => false,
10371                    BusyGaugeObservation::Omitted => {
10372                        // Once per drain: a module that never reports a gauge
10373                        // it declared would otherwise say so on every probe.
10374                        if !omission_reported {
10375                            forwarding
10376                                .counters()
10377                                .increment_drains_with_undeclared_gauge();
10378                            let missing = busy_gauges
10379                                .iter()
10380                                .zip(&verdicts)
10381                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10382                                .map(|(gauge, _)| gauge.as_str())
10383                                .collect::<Vec<_>>()
10384                                .join(",");
10385                            warn!(
10386                                module_id,
10387                                missing_gauges = %missing,
10388                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10389                            );
10390                            omission_reported = true;
10391                        }
10392                        false
10393                    }
10394                },
10395                Err(err) => {
10396                    warn!(
10397                        module_id,
10398                        error = %err,
10399                        "drain health.check did not produce declared busy gauges; treating module as busy"
10400                    );
10401                    false
10402                }
10403            };
10404            // Counted from the probe's completion: the probe above was awaited,
10405            // so no second probe can start while one is still outstanding.
10406            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10407        }
10408
10409        let in_flight = forwarding
10410            .endpoint_in_flight_count(endpoint)
10411            .map_err(SuperviseError::Forwarding)?;
10412        if in_flight == 0 && gauges_quiescent {
10413            return Ok(true);
10414        }
10415
10416        let now = Instant::now();
10417        if now >= deadline {
10418            return Ok(false);
10419        }
10420        let mut wait = deadline
10421            .saturating_duration_since(now)
10422            .min(REGISTRY_RELEASE_POLL);
10423        if !busy_gauges.is_empty() {
10424            wait = wait.min(next_probe_at.saturating_duration_since(now));
10425        }
10426        sleep(wait).await;
10427    }
10428}
10429
10430/// The wait between drain busy-gauge probes: the larger of
10431/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10432/// drain's own in-flight poll interval.
10433fn drain_busy_reprobe_interval() -> Duration {
10434    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10435}
10436
10437/// Logs why a drain gave up: the drain waited its full timeout without the
10438/// module going quiet, and teardown is being forced.
10439///
10440/// The line names two kinds of evidence, because a bare "timed out" hides
10441/// causes that need different fixes:
10442/// - `holdouts`: the requests and routes still open when the timeout hit,
10443///   and on which connections. One stuck call is a module bug; a held stream
10444///   on every session is a module that should end its streams when it
10445///   receives `module.draining`.
10446/// - `gauges`: for each busy gauge the module declares, its latest reading
10447///   and how many probes ran. This separates a module that kept reporting
10448///   itself busy, one whose health answer never contained the gauge, and one
10449///   that never answered the probes, which otherwise all look like a timeout
10450///   with nothing in flight.
10451fn warn_drain_timed_out(
10452    module_id: &str,
10453    drain_timeout: Duration,
10454    reason: RouteCloseReason,
10455    holdouts: &crate::forwarding::DrainHoldouts,
10456    gauges: &DrainGaugeTrace,
10457) {
10458    warn!(
10459        module_id = %module_id,
10460        waited = ?drain_timeout,
10461        ?reason,
10462        held_requests = holdouts.requests,
10463        held_routes = holdouts.routes,
10464        total_routes = holdouts.total_routes,
10465        top_connections = ?holdouts.top_connections,
10466        // `module_channel:corr`, so the module can find each held request
10467        // in its own log; capped, so `held_requests` is the full count.
10468        held = %holdouts
10469            .held
10470            .iter()
10471            .map(|(channel, corr)| format!("{channel}:{corr}"))
10472            .collect::<Vec<_>>()
10473            .join(","),
10474        gauge_probes = gauges.probes,
10475        busy_gauges = %gauges,
10476        "route drain timed out before request quiescence; forcing teardown"
10477    );
10478}
10479
10480/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10481///
10482/// `Ok` is always honest and passed straight through -- the wait actually measured
10483/// in-flight state. `Err` means the wait produced no measurement at all (the
10484/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10485/// constant: the drain did not complete. Never recomputed from route state, never a
10486/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10487fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10488    match wait_result {
10489        Ok(drained) => *drained,
10490        Err(_) => false,
10491    }
10492}
10493
10494fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10495    for released in released_routes {
10496        let frame = match Frame::build_with_version(
10497            released.negotiated_ver,
10498            FrameType::Goodbye,
10499            control_flags(),
10500            released.channel,
10501            released.epoch,
10502            0,
10503            Vec::new(),
10504        ) {
10505            Ok(frame) => frame,
10506            Err(err) => {
10507                warn!(
10508                    route_channel = released.channel,
10509                    error = %err,
10510                    "failed to build supervisor drain route GOODBYE frame"
10511                );
10512                continue;
10513            }
10514        };
10515        if !released.close_on_delivery_failure() {
10516            crate::forwarding::send_module_route_goodbye(
10517                &forwarding.counters(),
10518                &released.sink,
10519                frame,
10520                released.module_id.as_deref(),
10521                "supervisor drain",
10522            );
10523            continue;
10524        }
10525        if let Err(err) = released.sink.try_send(frame) {
10526            warn!(
10527                target_connection_id = released.connection_id.get(),
10528                route_channel = released.channel,
10529                error = %err,
10530                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10531            );
10532            let _ = forwarding.escalate_client_delivery_failure(
10533                released.connection_id,
10534                released.channel,
10535                released.epoch,
10536                CloseReason::new(
10537                    "route_goodbye_delivery_failed",
10538                    format!(
10539                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10540                        released.channel
10541                    ),
10542                ),
10543                crate::forwarding::UndeliveredFrame {
10544                    module_id: released.module_id.as_deref(),
10545                    sink: &released.sink,
10546                },
10547            );
10548        }
10549    }
10550}
10551
10552fn send_module_draining(
10553    module_id: &str,
10554    reason: RouteCloseReason,
10555    deadline_ms: u64,
10556    target: &ModuleDrainTarget,
10557) {
10558    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10559        reason,
10560        deadline_ms,
10561    }) {
10562        Ok(body) => body,
10563        Err(err) => {
10564            warn!(
10565                module_id,
10566                error = %err,
10567                "failed to encode module draining command"
10568            );
10569            return;
10570        }
10571    };
10572    let frame = match Frame::build_with_version(
10573        target.negotiated_ver,
10574        FrameType::Push,
10575        control_flags(),
10576        0,
10577        0,
10578        0,
10579        body,
10580    ) {
10581        Ok(frame) => frame,
10582        Err(err) => {
10583            warn!(
10584                module_id,
10585                error = %err,
10586                "failed to build module draining command frame"
10587            );
10588            return;
10589        }
10590    };
10591    if let Err(err) = target.sink.try_send(frame) {
10592        warn!(
10593            module_id,
10594            target_connection_id = target.endpoint.connection_id.get(),
10595            error = %err,
10596            "module draining command was not delivered to peer"
10597        );
10598    }
10599}
10600
10601/// The channel-0 GOODBYE that tells a module its stop is planned.
10602fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10603    match Frame::build_with_version(
10604        negotiated_ver,
10605        FrameType::Goodbye,
10606        control_flags(),
10607        0,
10608        0,
10609        0,
10610        Vec::new(),
10611    ) {
10612        Ok(frame) => Some(frame),
10613        Err(err) => {
10614            warn!(
10615                module_id,
10616                error = %err,
10617                "failed to build module GOODBYE frame"
10618            );
10619            None
10620        }
10621    }
10622}
10623
10624/// Send every registered module connection its module GOODBYE at daemon
10625/// shutdown, then request that connection's close.
10626///
10627/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10628/// before EOF, so the GOODBYE must reach the socket before the close. A close
10629/// request does not wait for the connection's queued frames: its writer gets a
10630/// bounded grace after the close, is aborted if it overruns it, and the daemon
10631/// process may exit before that grace ends. So with `wait_for_flush`, each
10632/// connection is closed only after its writer has acknowledged writing the
10633/// GOODBYE, or once a short shared budget runs out, so one module that is not
10634/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10635/// are only queued, for a shutdown the operator has told to stop waiting.
10636/// A connection that is already gone is skipped.
10637#[cfg(unix)]
10638async fn send_module_goodbyes_for_daemon_shutdown(
10639    forwarding: &Arc<ForwardingTable>,
10640    reason: &CloseReason,
10641    wait_for_flush: bool,
10642) {
10643    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10644    let targets = match forwarding.module_connections() {
10645        Ok(targets) => targets,
10646        Err(err) => {
10647            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10648            return;
10649        }
10650    };
10651    let deadline = Instant::now() + GOODBYE_BUDGET;
10652    let mut sends = tokio::task::JoinSet::new();
10653    for target in targets {
10654        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10655            continue;
10656        };
10657        if !wait_for_flush {
10658            if let Err(err) = target.sink.try_send(frame) {
10659                debug!(
10660                    module_id = %target.module_id,
10661                    error = %err,
10662                    "shutdown module GOODBYE was not queued"
10663                );
10664            }
10665            continue;
10666        }
10667        let forwarding = Arc::clone(forwarding);
10668        let reason = reason.clone();
10669        sends.spawn(async move {
10670            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10671                Ok(Ok(())) => {}
10672                Ok(Err(err)) => debug!(
10673                    module_id = %target.module_id,
10674                    error = %err,
10675                    "module connection closed before its shutdown GOODBYE was written"
10676                ),
10677                Err(_) => warn!(
10678                    module_id = %target.module_id,
10679                    budget = ?GOODBYE_BUDGET,
10680                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10681                ),
10682            }
10683            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10684        });
10685    }
10686    // Every task ends by the shared deadline, so this wait is bounded too.
10687    while sends.join_next().await.is_some() {}
10688}
10689
10690fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10691    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10692        return;
10693    };
10694    if let Err(err) = target.sink.try_send(frame) {
10695        warn!(
10696            module_id,
10697            target_connection_id = target.endpoint.connection_id.get(),
10698            error = %err,
10699            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10700        );
10701        forwarding.request_connection_close(
10702            target.endpoint.connection_id,
10703            CloseReason::new(
10704                "module_goodbye_delivery_failed",
10705                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10706            ),
10707        );
10708    }
10709}
10710
10711#[derive(Clone, Copy)]
10712struct ForwardingDrainContext<'a> {
10713    spec: &'a ModuleSpec,
10714    runtime: &'a SupervisorRuntimeConfig,
10715    registry: &'a Registry,
10716    scope: DrainScope,
10717}
10718
10719/// Which process a forwarding drain addresses.
10720#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10721enum DrainScope {
10722    /// Whatever endpoint is active for the module id: every plain stop,
10723    /// restart and reload. Also moves the module's state to `Draining`.
10724    Active,
10725    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10726    /// module id would resolve to the promoted candidate and leave neither
10727    /// process routable. The module's state is left alone, since the promoted
10728    /// candidate is what it describes and that process is running.
10729    Endpoint(crate::ModuleEndpointId),
10730}
10731
10732/// Whether a child being drained has already been asked to stop by the time
10733/// its drain wait starts.
10734///
10735/// The drain wait is the same budget whatever this says. What it decides is
10736/// whether the supervisor must ask by signal before that wait begins: a child
10737/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10738/// healthy or not.
10739#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10740enum StopNotice {
10741    /// The module was sent `module.draining` and a module GOODBYE over its own
10742    /// registered connection, and stops itself.
10743    SentOverConnection,
10744    /// The forwarding drain found no registered connection for the module: a
10745    /// subc child spawned moments ago that has not sent HELLO yet, or a
10746    /// `protocol: "none"` child, which never registers.
10747    NoConnection,
10748    /// This path sends nothing over the module's connection: the supervisor has
10749    /// no forwarding table, or the caller stops the child without a forwarding
10750    /// drain.
10751    NotSent,
10752}
10753
10754async fn begin_forwarding_drain(
10755    spec: &ModuleSpec,
10756    runtime: &SupervisorRuntimeConfig,
10757    registry: &Registry,
10758    snapshot: &SharedSnapshot,
10759    enabled: Option<bool>,
10760    reason: RouteCloseReason,
10761) -> Result<StopNotice, SuperviseError> {
10762    let Some(forwarding) = runtime.forwarding.as_ref() else {
10763        return Err(SuperviseError::ReloadUnavailable {
10764            module_id: spec.module_id.clone(),
10765            reason: "supervisor was not configured with a forwarding table".to_string(),
10766        });
10767    };
10768
10769    begin_forwarding_drain_with(
10770        forwarding,
10771        ForwardingDrainContext {
10772            spec,
10773            runtime,
10774            registry,
10775            scope: DrainScope::Active,
10776        },
10777        snapshot,
10778        enabled,
10779        reason,
10780        runtime.drain_timeout,
10781    )
10782    .await
10783}
10784
10785async fn begin_forwarding_drain_if_configured(
10786    spec: &ModuleSpec,
10787    runtime: &SupervisorRuntimeConfig,
10788    registry: &Registry,
10789    snapshot: &SharedSnapshot,
10790    enabled: Option<bool>,
10791    reason: RouteCloseReason,
10792) -> Result<StopNotice, SuperviseError> {
10793    begin_forwarding_drain_with_timeout(
10794        spec,
10795        runtime,
10796        registry,
10797        snapshot,
10798        enabled,
10799        reason,
10800        runtime.drain_timeout,
10801    )
10802    .await
10803}
10804
10805/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10806/// budget, for paths where the operator overrides the module's configured one
10807/// (`supervisor.restart{drain_timeout_ms}`).
10808async fn begin_forwarding_drain_with_timeout(
10809    spec: &ModuleSpec,
10810    runtime: &SupervisorRuntimeConfig,
10811    registry: &Registry,
10812    snapshot: &SharedSnapshot,
10813    enabled: Option<bool>,
10814    reason: RouteCloseReason,
10815    drain_timeout: Duration,
10816) -> Result<StopNotice, SuperviseError> {
10817    let Some(forwarding) = runtime.forwarding.as_ref() else {
10818        return Ok(StopNotice::NotSent);
10819    };
10820
10821    begin_forwarding_drain_with(
10822        forwarding,
10823        ForwardingDrainContext {
10824            spec,
10825            runtime,
10826            registry,
10827            scope: DrainScope::Active,
10828        },
10829        snapshot,
10830        enabled,
10831        reason,
10832        drain_timeout,
10833    )
10834    .await
10835}
10836
10837async fn begin_forwarding_drain_with(
10838    forwarding: &ForwardingTable,
10839    context: ForwardingDrainContext<'_>,
10840    snapshot: &SharedSnapshot,
10841    enabled: Option<bool>,
10842    reason: RouteCloseReason,
10843    drain_timeout: Duration,
10844) -> Result<StopNotice, SuperviseError> {
10845    let ForwardingDrainContext {
10846        spec,
10847        runtime,
10848        registry,
10849        scope,
10850    } = context;
10851    debug_assert_ne!(reason, RouteCloseReason::Crash);
10852    let terminal = matches!(reason, RouteCloseReason::Disable);
10853    let drain_started_at = Instant::now();
10854    let drain_deadline = drain_started_at + drain_timeout;
10855    let deadline_ms =
10856        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10857    let busy_gauges = match scope {
10858        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10859        DrainScope::Endpoint(endpoint) => {
10860            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10861        }
10862    };
10863
10864    // Admission gate first: route.open/commit and route REQUEST admission are closed
10865    // before the first quiescence check, so the outstanding count can only fall.
10866    let gate_started = Instant::now();
10867    let drain_target = match scope {
10868        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10869        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10870    }
10871    .map_err(SuperviseError::Forwarding)?;
10872    // The instant admission closed, and how long taking the forwarding write
10873    // lock to close it took. The timeout line reports only the quiescence
10874    // wait, so without this a drain that started late looked like one that
10875    // started on time.
10876    info!(
10877        module_id = %spec.module_id,
10878        ?reason,
10879        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10880        connected = drain_target.is_some(),
10881        "module drain began; route admission closed"
10882    );
10883    if scope == DrainScope::Active {
10884        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10885            state.state = ModuleState::Draining;
10886            state.draining_to_replace =
10887                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10888            if let Some(enabled) = enabled {
10889                state.enabled = enabled;
10890            }
10891        })?;
10892    }
10893
10894    let Some(target) = drain_target.as_ref() else {
10895        // Nothing was sent: the module has no registered connection to carry
10896        // `module.draining` or a GOODBYE. The caller must not assume the child
10897        // was asked to stop.
10898        return Ok(StopNotice::NoConnection);
10899    };
10900    {
10901        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10902        #[cfg(feature = "test-support")]
10903        forwarding.wait_before_route_closing_for_test().await;
10904        let routes = forwarding
10905            .endpoint_routes(target.endpoint)
10906            .map_err(SuperviseError::Forwarding)?;
10907        let routes_notified = routes.len();
10908        crate::control::send_route_control_pushes(
10909            forwarding,
10910            routes.clone(),
10911            ClientControlPush::RouteClosing {
10912                module_id: spec.module_id.clone(),
10913                channels: Vec::new(),
10914                reason,
10915            },
10916        );
10917        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10918
10919        // `route.closing` was just sent above: from here on every return path,
10920        // including an early one, MUST send `route.closed` before propagating
10921        // anything else. A client holds `closing` as a promise that a verdict is
10922        // coming; leaving early without `closed` strands it waiting forever, since
10923        // `closing` carries no timeout of its own.
10924        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10925        let wait_result = wait_for_forwarding_quiescence(
10926            forwarding,
10927            &spec.module_id,
10928            runtime,
10929            target.endpoint,
10930            drain_deadline,
10931            &busy_gauges,
10932            scope,
10933            &mut gauge_trace,
10934        )
10935        .await;
10936        let drained = drained_after_quiescence_wait(&wait_result);
10937        if let Err(err) = &wait_result {
10938            error!(
10939                module_id = %spec.module_id,
10940                ?reason,
10941                error = %err,
10942                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10943            );
10944        } else if !drained {
10945            // Read before teardown releases the routes.
10946            let holdouts = forwarding
10947                .endpoint_drain_holdouts(target.endpoint)
10948                .unwrap_or_default();
10949            warn_drain_timed_out(
10950                &spec.module_id,
10951                drain_timeout,
10952                reason,
10953                &holdouts,
10954                &gauge_trace,
10955            );
10956        }
10957        crate::control::send_route_control_pushes(
10958            forwarding,
10959            routes,
10960            ClientControlPush::RouteClosed {
10961                module_id: spec.module_id.clone(),
10962                channels: Vec::new(),
10963                reason,
10964                drained,
10965                abandoned: target.abandoned_bindings.len() as u32,
10966                excluded_subscriptions: target.excluded_subscriptions,
10967                terminal: Some(terminal),
10968            },
10969        );
10970        wait_result?;
10971
10972        // `route.closed` has now been sent unconditionally above. From here the
10973        // remaining steps are cleanup (route + module GOODBYE) rather than a
10974        // promise the client is waiting on, but a lock-poisoned
10975        // `release_module_endpoint_routes` would otherwise skip the module
10976        // GOODBYE silently too -- send it before propagating the error.
10977        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10978            Ok(routes) => routes,
10979            Err(err) => {
10980                warn!(
10981                    module_id = %spec.module_id,
10982                    ?reason,
10983                    error = %err,
10984                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10985                );
10986                send_module_goodbye(&spec.module_id, forwarding, target);
10987                return Err(SuperviseError::Forwarding(err));
10988            }
10989        };
10990        let route_goodbye_count = released_routes.len();
10991        send_route_goodbyes(forwarding, released_routes);
10992        send_module_goodbye(&spec.module_id, forwarding, target);
10993
10994        // The drain's happy path was previously silent: every emission above is
10995        // best-effort with only its failure arm logged, so "were consumers told"
10996        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10997        // hang where the open question was exactly whether teardown notice went
10998        // out). One summary line makes that class decidable in one grep.
10999        info!(
11000            module_id = %spec.module_id,
11001            ?reason,
11002            routes_notified,
11003            route_goodbyes = route_goodbye_count,
11004            abandoned_reservations = target.abandoned_bindings.len(),
11005            excluded_subscriptions = target.excluded_subscriptions,
11006            drained,
11007            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11008        );
11009    }
11010
11011    Ok(StopNotice::SentOverConnection)
11012}
11013
11014/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11015/// the only slot a plain (non-swap) spawn can register into.
11016async fn wait_for_registration_after_reload(
11017    registry: &Registry,
11018    module_id: &str,
11019    snapshot: &SharedSnapshot,
11020    child: &mut SupervisedChild,
11021    wait: Duration,
11022) -> Result<RegistrationWaitOutcome, SuperviseError> {
11023    wait_for_slot_registration(
11024        registry,
11025        crate::registry::RegistrationSlot::Active(module_id),
11026        module_id,
11027        snapshot,
11028        child,
11029        wait,
11030    )
11031    .await
11032}
11033
11034/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11035///
11036/// Keyed on the slot rather than the bare module id because during a swap the
11037/// id's active slot is already held by the incumbent: an id-keyed wait would
11038/// report the incumbent's registration as the candidate's and a candidate that
11039/// never registers would look registered. A swap candidate waits on
11040/// `crate::registry::RegistrationSlot::Candidate`.
11041async fn wait_for_slot_registration(
11042    registry: &Registry,
11043    slot: crate::registry::RegistrationSlot<'_>,
11044    module_id: &str,
11045    snapshot: &SharedSnapshot,
11046    child: &mut SupervisedChild,
11047    wait: Duration,
11048) -> Result<RegistrationWaitOutcome, SuperviseError> {
11049    let deadline = Instant::now() + wait;
11050    loop {
11051        if registry
11052            .registration(slot)
11053            .map_err(SuperviseError::Registry)?
11054            .is_some()
11055        {
11056            return Ok(RegistrationWaitOutcome::Registered);
11057        }
11058
11059        let now = Instant::now();
11060        if now >= deadline {
11061            return Ok(RegistrationWaitOutcome::TimedOut);
11062        }
11063        let remaining = deadline.saturating_duration_since(now);
11064        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11065
11066        tokio::select! {
11067            wait_result = child.wait() => {
11068                let status = wait_result.map_err(|source| SuperviseError::Wait {
11069                    module_id: module_id.to_string(),
11070                    source,
11071                })?;
11072                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11073                    snapshot,
11074                    child,
11075                    &status,
11076                )));
11077            }
11078            _ = sleep(poll) => {}
11079        }
11080    }
11081}
11082
11083fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11084    // A process stopped for missing its first HELLO did not provide service,
11085    // even if its TERM handler exits zero. Reload also uses this classification
11086    // when a replacement exits before registering.
11087    if exit_report.kind != ExitKind::DeliberateSeverance {
11088        exit_report.kind = ExitKind::Crash;
11089    }
11090    exit_report
11091}
11092
11093async fn handle_reload_child_registration_failure(
11094    spec: &ModuleSpec,
11095    runtime: &SupervisorRuntimeConfig,
11096    registry: &Registry,
11097    process_liveness: &SupervisorProcessLiveness,
11098    snapshot: &SharedSnapshot,
11099    _child: &mut Option<SupervisedChild>,
11100    failure: ReloadRegistrationFailure,
11101) -> Result<(), SuperviseError> {
11102    let ReloadRegistrationFailure {
11103        exit_report,
11104        reason,
11105    } = failure;
11106    match on_child_exit(
11107        spec,
11108        runtime.restart_policy,
11109        registry,
11110        snapshot,
11111        &runtime.terminal_ring,
11112        &runtime.spawn_events,
11113        &runtime.child_roster,
11114        exit_report,
11115        runtime.recovery_is_daemon,
11116    )
11117    .await
11118    {
11119        NextAction::Stop {
11120            registration_released,
11121        } => {
11122            if registration_released {
11123                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11124            }
11125        }
11126        NextAction::Restart { schedule } => {
11127            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11128                schedule.delay
11129            });
11130            if let Some(schedule) = schedule {
11131                log_crash_respawn(&spec.module_id, schedule);
11132            }
11133            schedule_respawn(
11134                runtime,
11135                snapshot,
11136                &spec.module_id,
11137                delay,
11138                RespawnKind::Spawn,
11139                true,
11140            )?;
11141        }
11142    }
11143    Err(SuperviseError::ReloadFailed {
11144        module_id: spec.module_id.clone(),
11145        reason,
11146    })
11147}
11148
11149async fn handle_reload_spawn_failure(
11150    spec: &ModuleSpec,
11151    runtime: &SupervisorRuntimeConfig,
11152    process_liveness: &SupervisorProcessLiveness,
11153    snapshot: &SharedSnapshot,
11154    _child: &mut Option<SupervisedChild>,
11155    reason: String,
11156) -> Result<(), SuperviseError> {
11157    let now = Instant::now();
11158    let mut schedule = None;
11159    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11160        clear_current_process_facts(state);
11161        if state.enabled {
11162            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11163            state.state = if schedule.is_some() {
11164                ModuleState::Restarting
11165            } else {
11166                if runtime
11167                    .failure_context(runtime.recovery_is_daemon)
11168                    .may_retry()
11169                {
11170                    state.failed_retry.schedule(&spec.module_id);
11171                }
11172                ModuleState::Failed
11173            };
11174        } else {
11175            state.state = ModuleState::Disabled;
11176        }
11177    })?;
11178    if let Some(schedule) = schedule {
11179        schedule_respawn(
11180            runtime,
11181            snapshot,
11182            &spec.module_id,
11183            schedule.delay,
11184            RespawnKind::Spawn,
11185            true,
11186        )?;
11187    } else {
11188        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11189    }
11190    Err(SuperviseError::ReloadFailed {
11191        module_id: spec.module_id.clone(),
11192        reason,
11193    })
11194}
11195
11196fn control_flags() -> Flags {
11197    Flags::new(false, Priority::Passive, false)
11198}
11199
11200#[allow(clippy::too_many_arguments)]
11201async fn drain_optional_child(
11202    module_id: &str,
11203    protocol: ModuleProtocol,
11204    stop_notice: StopNotice,
11205    registry: &Registry,
11206    forwarding: Option<&ForwardingTable>,
11207    snapshot: &SharedSnapshot,
11208    terminal_ring: &Arc<Mutex<TerminalRing>>,
11209    spawn_events: &SpawnEventFeed,
11210    child: &mut Option<SupervisedChild>,
11211    drain_timeout: Duration,
11212    final_state: ModuleState,
11213    enabled: Option<bool>,
11214    failure: &FailureContext,
11215) -> Result<(), SuperviseError> {
11216    if let Some(child) = child.take() {
11217        drain_child_to_state(
11218            module_id,
11219            protocol,
11220            stop_notice,
11221            registry,
11222            forwarding,
11223            snapshot,
11224            terminal_ring,
11225            spawn_events,
11226            child,
11227            drain_timeout,
11228            final_state,
11229            enabled,
11230            failure,
11231        )
11232        .await
11233    } else {
11234        update_snapshot(snapshot, Some(module_id), |state| {
11235            state.state = final_state;
11236            if final_state == ModuleState::Failed {
11237                state.failed_retry.failed_at = Some(Instant::now());
11238            }
11239            if let Some(enabled) = enabled {
11240                state.enabled = enabled;
11241            }
11242            clear_current_process_facts(state);
11243        })?;
11244        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11245    }
11246}
11247
11248#[allow(clippy::too_many_arguments)]
11249async fn drain_child_to_state(
11250    module_id: &str,
11251    _protocol: ModuleProtocol,
11252    stop_notice: StopNotice,
11253    registry: &Registry,
11254    forwarding: Option<&ForwardingTable>,
11255    snapshot: &SharedSnapshot,
11256    terminal_ring: &Arc<Mutex<TerminalRing>>,
11257    spawn_events: &SpawnEventFeed,
11258    mut child: SupervisedChild,
11259    drain_timeout: Duration,
11260    final_state: ModuleState,
11261    enabled: Option<bool>,
11262    failure: &FailureContext,
11263) -> Result<(), SuperviseError> {
11264    let protocol = child.protocol;
11265    update_snapshot(snapshot, Some(module_id), |state| {
11266        state.state = ModuleState::Draining;
11267        state.draining_to_replace = final_state == ModuleState::Restarting;
11268        if let Some(enabled) = enabled {
11269            state.enabled = enabled;
11270        }
11271    })?;
11272
11273    // The wait below is the same budget in every case; what differs is
11274    // whether anything has ASKED the child to stop before it starts. Only a
11275    // forwarding drain that reached the module's registered connection has
11276    // (`module.draining`, then a module GOODBYE). Every other child was told
11277    // nothing: a `protocol: "none"` module, which never registers; a subc
11278    // module spawned moments ago that has not sent HELLO yet; or a stop that
11279    // runs no forwarding drain. Without a signal the budget is only a delay
11280    // in front of SIGKILL -- and the not-yet-registered child is the worst
11281    // case, because it registers into a module that is already draining,
11282    // is never told, and is killed while healthy.
11283    if stop_notice != StopNotice::SentOverConnection {
11284        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11285            info!(
11286                module_id,
11287                pid = child.pid,
11288                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11289                "module has no connection yet; requesting stop by signal"
11290            );
11291        }
11292        request_graceful_stop(module_id, &child);
11293    }
11294
11295    let exit_report = match timeout(drain_timeout, child.wait()).await {
11296        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11297        Ok(Err(source)) => {
11298            fail_after_wait_error(
11299                module_id,
11300                snapshot,
11301                terminal_ring,
11302                spawn_events,
11303                format!("failed to wait during drain: {source}"),
11304                failure,
11305            );
11306            return Err(SuperviseError::Wait {
11307                module_id: module_id.to_string(),
11308                source,
11309            });
11310        }
11311        Err(_) => {
11312            // Mirror the sibling arm above: state is already `Draining`, and an
11313            // error propagated from here would strand it there -- a state
11314            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11315            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11316            // `Failed` before `?` keeps the module operator-visible and
11317            // revivable. Trigger is an ESRCH race (process exits between the
11318            // drain timeout firing and the kill) or a post-kill wait failure
11319            // (issue #34).
11320            //
11321            // Logged because the kill is otherwise visible only as signal 9 in
11322            // the terminal ring, and the budget it follows can be long enough
11323            // that consumers see a stretch of refusals with no stated cause.
11324            warn!(
11325                module_id,
11326                pid = child.pid,
11327                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11328                reason = ?final_state,
11329                ?stop_notice,
11330                "drain budget expired before the module exited; killing it"
11331            );
11332            child.start_kill().map_err(|source| {
11333                fail_after_wait_error(
11334                    module_id,
11335                    snapshot,
11336                    terminal_ring,
11337                    spawn_events,
11338                    format!("failed to kill during drain: {source}"),
11339                    failure,
11340                );
11341                SuperviseError::Kill {
11342                    module_id: module_id.to_string(),
11343                    source,
11344                }
11345            })?;
11346            let status = child.wait().await.map_err(|source| {
11347                fail_after_wait_error(
11348                    module_id,
11349                    snapshot,
11350                    terminal_ring,
11351                    spawn_events,
11352                    format!("failed to wait after drain kill: {source}"),
11353                    failure,
11354                );
11355                SuperviseError::Wait {
11356                    module_id: module_id.to_string(),
11357                    source,
11358                }
11359            })?;
11360            classify_reaped_child_exit(snapshot, &child, &status)
11361        }
11362    };
11363
11364    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11365    publish_after_terminal_record(
11366        module_id,
11367        || {
11368            record_terminal_with_detail(
11369                module_id,
11370                terminal_ring,
11371                spawn_events,
11372                &exit_report,
11373                terminal_disposition(final_state),
11374                detail,
11375            )
11376        },
11377        || {
11378            update_snapshot(snapshot, Some(module_id), |state| {
11379                state.state = final_state;
11380                if final_state == ModuleState::Failed {
11381                    state.failed_retry.failed_at = Some(Instant::now());
11382                }
11383                if let Some(enabled) = enabled {
11384                    state.enabled = enabled;
11385                }
11386                clear_current_process_facts(state);
11387                state.last_exit = Some(exit_report.clone());
11388                if exit_report.kind == ExitKind::DeliberateSeverance {
11389                    state.lifetime_restarts += 1;
11390                }
11391            })
11392        },
11393    )?;
11394    child.drain_stderr(module_id).await;
11395
11396    release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11397}
11398
11399/// Ask a child that nothing else has asked to stop, by signal.
11400///
11401/// A registered subc module is asked over its own connection: the drain sends
11402/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11403/// module GOODBYE, and the module stops itself. A module that speaks no subc
11404/// wire receives none of that, and neither does a subc module that has not
11405/// registered yet, so for them the drain budget would be pure delay in front of
11406/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11407/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11408/// into a recovery on the next start.
11409///
11410/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11411/// rule rather than an optimisation: that module's graceful stop is already
11412/// running by the time its child is drained, and a signal would race it.
11413///
11414/// Best-effort by construction. A child that has already exited is the ordinary
11415/// case rather than an error (the kill lands on a reaped or exiting pid), so a
11416/// failure is logged at debug and the wait-then-kill below still decides the
11417/// outcome.
11418#[cfg(unix)]
11419fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11420    let Some(pid) = child
11421        .id()
11422        .and_then(|pid| i32::try_from(pid).ok())
11423        .and_then(rustix::process::Pid::from_raw)
11424    else {
11425        debug!(
11426            module_id,
11427            "no pid to signal for teardown; falling through to the drain wait"
11428        );
11429        return;
11430    };
11431    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11432        Ok(()) => debug!(
11433            module_id,
11434            "sent SIGTERM to a module nothing else asked to stop"
11435        ),
11436        Err(err) => debug!(
11437            module_id,
11438            error = %err,
11439            "SIGTERM to module failed; the drain wait and kill still apply"
11440        ),
11441    }
11442}
11443
11444/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11445/// Windows does offer need cooperation this supervisor cannot assume: a console
11446/// control event requires sharing a console with the child, and `WM_CLOSE`
11447/// requires the child to pump a message loop. A supervised server process does
11448/// neither, so there is nothing to send and teardown is the wait followed by the
11449/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11450/// the thing `protocol: "none"` exists to avoid.
11451#[cfg(not(unix))]
11452fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11453    debug!(
11454        module_id,
11455        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11456    );
11457}
11458
11459fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11460    match final_state {
11461        ModuleState::Stopped => TerminalDisposition::Stopped,
11462        ModuleState::Disabled => TerminalDisposition::Disabled,
11463        ModuleState::Restarting => TerminalDisposition::Restarting,
11464        ModuleState::Failed => TerminalDisposition::Failed,
11465        ModuleState::Starting
11466        | ModuleState::Running
11467        | ModuleState::Unresponsive
11468        | ModuleState::Draining => {
11469            unreachable!("terminal exits only finish in terminal or restarting states")
11470        }
11471    }
11472}
11473
11474/// Release a reaped child's registration before allowing another spawn.
11475///
11476/// EOF is not a process-lifetime signal: an inherited socket can stay open
11477/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11478/// reading EOF. After the normal release grace, request connection close (which
11479/// cancels both reads and dispatch), then allow one more release grace for the
11480/// connection guard's forwarding cleanup. Never evict a different connection.
11481async fn release_dead_registration(
11482    registry: &Registry,
11483    forwarding: Option<&ForwardingTable>,
11484    snapshot: &SharedSnapshot,
11485    module_id: &str,
11486    failure: &FailureContext,
11487) -> Result<(), SuperviseError> {
11488    let result = async {
11489        let registration = registry
11490            .get_module(module_id)
11491            .map_err(SuperviseError::Registry)?;
11492        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11493            Ok(()) => return Ok(()),
11494            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11495            Err(err) => return Err(err),
11496        }
11497        let pid = lock_snapshot(snapshot)?.reaped_pid;
11498        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11499            warn!(
11500                module_id,
11501                pid,
11502                connection_id = registration.connection_id.get(),
11503                "reaped module registration outlived release grace; closing dead connection"
11504            );
11505            forwarding.request_connection_close(
11506                registration.connection_id,
11507                CloseReason::new(
11508                    "supervised_process_reaped",
11509                    format!("module '{module_id}' pid {pid} exited"),
11510                ),
11511            );
11512            wait_for_slot_registration_release(
11513                registry,
11514                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11515                REGISTRY_RELEASE_TIMEOUT,
11516            )
11517            .await?;
11518        }
11519        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11520    }
11521    .await;
11522    if let Err(err) = &result {
11523        fail_snapshot(snapshot, Some(module_id), None, failure);
11524        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11525    }
11526    result
11527}
11528
11529/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11530/// plain stop or restart waits for before it spawns a replacement.
11531async fn wait_for_registration_release(
11532    registry: &Registry,
11533    module_id: &str,
11534    wait: Duration,
11535) -> Result<(), SuperviseError> {
11536    wait_for_slot_registration_release(
11537        registry,
11538        crate::registry::RegistrationSlot::Active(module_id),
11539        wait,
11540    )
11541    .await
11542}
11543
11544/// Wait for the registration in `slot` to go away.
11545///
11546/// Keyed on the slot rather than the bare module id because a successful swap
11547/// never empties the id's active slot (the promoted candidate is in it), so an
11548/// id-keyed wait for the incumbent's release would always time out. Draining a
11549/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11550/// incumbent's connection instead.
11551async fn wait_for_slot_registration_release(
11552    registry: &Registry,
11553    slot: crate::registry::RegistrationSlot<'_>,
11554    wait: Duration,
11555) -> Result<(), SuperviseError> {
11556    let deadline = Instant::now() + wait;
11557    let mut release_events = registration_release_events().subscribe();
11558    let still_active = |registration: &crate::registry::ModuleRegistration| {
11559        SuperviseError::RegistrationStillActive {
11560            module_id: registration.manifest.module_id.clone(),
11561            waited: wait,
11562        }
11563    };
11564    loop {
11565        let _observed_generation = *release_events.borrow_and_update();
11566        let Some(registration) = registry
11567            .registration(slot)
11568            .map_err(SuperviseError::Registry)?
11569        else {
11570            return Ok(());
11571        };
11572
11573        let now = Instant::now();
11574        if now >= deadline {
11575            return Err(still_active(&registration));
11576        }
11577
11578        let remaining = deadline.saturating_duration_since(now);
11579        match timeout(remaining, release_events.changed()).await {
11580            Ok(Ok(())) | Ok(Err(_)) => {}
11581            Err(_) => return Err(still_active(&registration)),
11582        }
11583    }
11584}
11585
11586#[cfg(test)]
11587mod slot_registration_wait_tests {
11588    use super::*;
11589    use crate::registry::{ConnectionId, RegistrationSlot};
11590    use subc_protocol::manifest::ModuleManifest;
11591
11592    #[tokio::test]
11593    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11594        let registry = Arc::new(Registry::default());
11595        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11596        let runtime = supervisor.runtime_config();
11597        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11598        let spec = ModuleSpec {
11599            module_id: "enable-stale-registration".to_string(),
11600            program: PathBuf::from("/missing/enable-retry-test"),
11601            args: Vec::new(),
11602            env: Vec::new(),
11603            reserved: false,
11604            reserved_prefixes: Vec::new(),
11605            protocol: ModuleProtocol::Subc,
11606            overlap: Default::default(),
11607        };
11608        let connection = ConnectionId::new(90);
11609        registry
11610            .register_with_control_ops(
11611                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11612                1,
11613                connection,
11614                Vec::new(),
11615            )
11616            .unwrap();
11617        let mut child = None;
11618        let err = set_child_enabled(
11619            &spec,
11620            &runtime,
11621            &registry,
11622            &supervisor.process_liveness,
11623            &snapshot,
11624            &mut child,
11625            true,
11626            &runtime.failure_context(false),
11627        )
11628        .await
11629        .unwrap_err();
11630        assert!(matches!(
11631            err,
11632            SuperviseError::RegistrationStillActive { .. }
11633        ));
11634        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11635        assert!(child.is_none());
11636        registry.deregister_connection(connection).unwrap();
11637        let err = set_child_enabled(
11638            &spec,
11639            &runtime,
11640            &registry,
11641            &supervisor.process_liveness,
11642            &snapshot,
11643            &mut child,
11644            true,
11645            &runtime.failure_context(false),
11646        )
11647        .await
11648        .unwrap_err();
11649        assert!(
11650            matches!(err, SuperviseError::Spawn { .. }),
11651            "second enable must attempt a spawn: {err}"
11652        );
11653        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11654    }
11655
11656    const INCUMBENT: u64 = 1;
11657    const CANDIDATE: u64 = 2;
11658
11659    fn swapped_registry() -> Arc<Registry> {
11660        let registry = Arc::new(Registry::default());
11661        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11662        registry
11663            .register_with_control_ops(
11664                manifest.clone(),
11665                1,
11666                ConnectionId::new(INCUMBENT),
11667                Vec::new(),
11668            )
11669            .unwrap();
11670        registry
11671            .register_candidate_with_control_ops(
11672                manifest,
11673                1,
11674                ConnectionId::new(CANDIDATE),
11675                Vec::new(),
11676            )
11677            .unwrap();
11678        registry
11679    }
11680
11681    /// After a promotion the id's active slot is held by the new process, so an
11682    /// id-keyed wait for the incumbent's release can never succeed; the
11683    /// connection-keyed wait completes as soon as the incumbent deregisters.
11684    #[tokio::test]
11685    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11686        let registry = swapped_registry();
11687        registry.promote_candidate("m").unwrap().unwrap();
11688
11689        assert!(matches!(
11690            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11691            Err(SuperviseError::RegistrationStillActive { .. })
11692        ));
11693
11694        // Still held while the incumbent's connection has not deregistered.
11695        assert!(matches!(
11696            wait_for_slot_registration_release(
11697                &registry,
11698                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11699                Duration::from_millis(50),
11700            )
11701            .await,
11702            Err(SuperviseError::RegistrationStillActive { .. })
11703        ));
11704
11705        let releaser = Arc::clone(&registry);
11706        let release = tokio::spawn(async move {
11707            sleep(Duration::from_millis(20)).await;
11708            releaser
11709                .deregister_connection(ConnectionId::new(INCUMBENT))
11710                .unwrap();
11711            notify_registration_release();
11712        });
11713        wait_for_slot_registration_release(
11714            &registry,
11715            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11716            Duration::from_secs(5),
11717        )
11718        .await
11719        .expect("the incumbent's own registration is released");
11720        release.await.unwrap();
11721        assert!(registry.get_module("m").unwrap().is_some());
11722    }
11723
11724    /// The candidate slot is waited on separately from the active slot: the
11725    /// incumbent's registration neither holds up nor stands in for it.
11726    #[tokio::test]
11727    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11728        let registry = swapped_registry();
11729        assert!(matches!(
11730            wait_for_slot_registration_release(
11731                &registry,
11732                RegistrationSlot::Candidate("m"),
11733                Duration::from_millis(50),
11734            )
11735            .await,
11736            Err(SuperviseError::RegistrationStillActive { .. })
11737        ));
11738        registry
11739            .deregister_connection(ConnectionId::new(CANDIDATE))
11740            .unwrap();
11741        wait_for_slot_registration_release(
11742            &registry,
11743            RegistrationSlot::Candidate("m"),
11744            Duration::from_millis(50),
11745        )
11746        .await
11747        .expect("a candidate slot with no candidate is released");
11748        assert!(registry
11749            .registration(RegistrationSlot::Active("m"))
11750            .unwrap()
11751            .is_some());
11752    }
11753}
11754
11755fn classify_exit(status: &ExitStatus) -> ExitReport {
11756    ExitReport {
11757        kind: if status.success() {
11758            ExitKind::Clean
11759        } else {
11760            ExitKind::Crash
11761        },
11762        code: status.code(),
11763        signal: exit_signal(status),
11764        at_ms: unix_ms_now(),
11765    }
11766}
11767
11768/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11769/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11770/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11771/// disposition still must be `Failed` so the terminal ring is not silently missing
11772/// an entry, matching what `fail_snapshot` records for this same arm.
11773fn wait_error_exit_report() -> ExitReport {
11774    ExitReport {
11775        kind: ExitKind::Crash,
11776        code: None,
11777        signal: None,
11778        at_ms: unix_ms_now(),
11779    }
11780}
11781
11782#[cfg(unix)]
11783fn exit_signal(status: &ExitStatus) -> Option<i32> {
11784    use std::os::unix::process::ExitStatusExt;
11785
11786    status.signal()
11787}
11788
11789#[cfg(not(unix))]
11790fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11791    None
11792}
11793
11794/// Give an operator-touched module its full crash budget back.
11795///
11796/// Named for the counter it used to zero; it now empties the in-window ring,
11797/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11798/// ledger of what happened survives every operator action.
11799fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11800    update_snapshot(snapshot, Some(module_id), |state| {
11801        state.clear_crash_restarts();
11802    })
11803}
11804
11805fn set_running(
11806    snapshot: &SharedSnapshot,
11807    child: &SupervisedChild,
11808    module_id: &str,
11809    spawn_events: &SpawnEventFeed,
11810) -> Result<(), SuperviseError> {
11811    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11812        module_id: Some(module_id.to_string()),
11813    })?;
11814    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11815    if std::mem::take(&mut state.coalesced_restart_pending) {
11816        let generation = state.spawn_generation;
11817        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11818    }
11819    state.drain_disposition_detail = None;
11820    state.spawn_failure = None;
11821    state.waiting_for_previous_pid = None;
11822    state.waiting_for_previous_detail = None;
11823    // Every caller of this is a plain spawn, which always uses the primary key;
11824    // a promoted swap candidate sets the flag itself after this returns.
11825    state.in_alternate_slot = false;
11826    state.configuration_updated_since_spawn = false;
11827    state.spawned_protocol = Some(child.protocol);
11828    state.state = ModuleState::Running;
11829    state.enabled = true;
11830    state.process_alive = true;
11831    state.pid = child.id();
11832    // Probe samples belong to one process. Keep module-level escalation and
11833    // late-answer history, but never charge a new child for its predecessor's misses.
11834    state.health.status = SupervisorHealthStatus::Unknown;
11835    state.health.consecutive_failures = 0;
11836    state.health.detail = None;
11837    state.health.metrics = None;
11838    state.health.last_probe_ms = None;
11839    state.failed_retry.healthy_since = None;
11840    state.failed_retry.failed_at = None;
11841    state.failed_retry.running_since = Some(child.spawned_at);
11842    #[cfg(windows)]
11843    {
11844        state.pipe_consumed = child
11845            .nonce_delivery
11846            .as_ref()
11847            .map(|delivery| delivery.consumed());
11848    }
11849    #[cfg(target_os = "macos")]
11850    {
11851        state.report_ready = Some(Arc::clone(&child.report_ready));
11852    }
11853    state.spawned_at_ms = Some(child.spawned_at_ms);
11854    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11855    state.spawned_from = Some(child.spawned_from.clone());
11856    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11857    state.spawned_file_identity = child.spawned_file_identity.clone();
11858    state.process_start_time = child.process_start_time;
11859    Ok(())
11860}
11861
11862fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11863    state.failed_retry.healthy_since = None;
11864    state.failed_retry.running_since = None;
11865    state.waiting_for_previous_pid = None;
11866    state.waiting_for_previous_detail = None;
11867    state.process_alive = false;
11868    state.spawned_protocol = None;
11869    state.pid = None;
11870    #[cfg(windows)]
11871    {
11872        state.pipe_consumed = None;
11873    }
11874    #[cfg(target_os = "macos")]
11875    {
11876        state.report_ready = None;
11877    }
11878    state.spawned_at_ms = None;
11879    state.exec_outcome = None;
11880    state.spawned_from = None;
11881    state.launch_spec_digest = None;
11882    state.spawned_file_identity = None;
11883    state.process_start_time = None;
11884    state.deliberate_severance = None;
11885}
11886
11887#[cfg(test)]
11888fn record_deliberate_severance(
11889    snapshot: &SharedSnapshot,
11890    identity: ProcessIdentity,
11891) -> Result<(), SuperviseError> {
11892    update_snapshot(snapshot, None, |state| {
11893        state.deliberate_severance = Some(identity);
11894    })
11895}
11896
11897fn apply_deliberate_severance_marker(
11898    snapshot: &SharedSnapshot,
11899    exited_identity: Option<ProcessIdentity>,
11900    mut exit_report: ExitReport,
11901) -> ExitReport {
11902    let marker = lock_snapshot(snapshot)
11903        .ok()
11904        .and_then(|mut state| state.deliberate_severance.take());
11905    if marker.is_some() && marker == exited_identity {
11906        exit_report.kind = ExitKind::DeliberateSeverance;
11907    }
11908    exit_report
11909}
11910
11911fn classify_reaped_child_exit(
11912    snapshot: &SharedSnapshot,
11913    child: &SupervisedChild,
11914    status: &ExitStatus,
11915) -> ExitReport {
11916    let _ = update_snapshot(snapshot, None, |state| {
11917        state.reaped_pid = Some(child.pid);
11918        state.spawn_failure = child.spawn_failure.clone();
11919    });
11920    if child.spawn_failure.as_deref() == Some("registration_timeout") {
11921        registration_failure_exit_report(classify_exit(status))
11922    } else {
11923        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11924    }
11925}
11926
11927fn fail_snapshot(
11928    snapshot: &SharedSnapshot,
11929    module_id: Option<&str>,
11930    last_exit: Option<ExitReport>,
11931    failure: &FailureContext,
11932) {
11933    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11934        state.state = ModuleState::Failed;
11935        clear_current_process_facts(state);
11936        if failure.may_retry() && state.enabled {
11937            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11938        } else if failure.shutdown.is_set() {
11939            state.failed_retry.cancel();
11940        }
11941        if let Some(last_exit) = last_exit {
11942            state.last_exit = Some(last_exit);
11943        }
11944    }) {
11945        error!(error = %err, "failed to mark supervisor state failed");
11946    }
11947}
11948
11949fn update_snapshot(
11950    snapshot: &SharedSnapshot,
11951    module_id: Option<&str>,
11952    update: impl FnOnce(&mut SupervisorSnapshot),
11953) -> Result<(), SuperviseError> {
11954    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11955        module_id: module_id.map(ToOwned::to_owned),
11956    })?;
11957    update(&mut state);
11958    Ok(())
11959}
11960
11961const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11962
11963fn lock_snapshot_for_control<'a>(
11964    snapshot: &'a SharedSnapshot,
11965    module_id: &str,
11966    caller: &'static str,
11967) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11968    let started_at = Instant::now();
11969    let guard = lock_snapshot(snapshot)?;
11970    let waited = started_at.elapsed();
11971    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11972        warn!(
11973            module_id = %module_id,
11974            waited_ms = waited.as_millis() as u64,
11975            caller = %caller,
11976            "slow snapshot lock"
11977        );
11978    }
11979    Ok(guard)
11980}
11981
11982fn lock_snapshot(
11983    snapshot: &SharedSnapshot,
11984) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11985    snapshot
11986        .lock()
11987        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11988}
11989
11990#[cfg(test)]
11991mod terminal_history_tests {
11992    use std::{
11993        path::PathBuf,
11994        sync::Arc,
11995        time::{Duration, Instant},
11996    };
11997
11998    use tokio::{sync::mpsc, time::sleep};
11999
12000    use super::{
12001        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12002        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12003        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12004        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12005        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12006        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12007        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12008        SupervisorHealthStatus, SupervisorSnapshot,
12009    };
12010    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12011    // use for their own wall-clock deadlines: crash-restart instants must be on
12012    // the same clock the production code stamps them with, which is tokio's (and
12013    // is what `start_paused` tests can move).
12014    use super::Instant as ClockInstant;
12015    use crate::{
12016        registry::Registry,
12017        terminal_ring::{TerminalRing, TerminalRingConfig},
12018    };
12019    use std::sync::Mutex;
12020    use subc_control::TerminalDisposition;
12021
12022    /// See the twin in `control.rs` for why this derives the path from
12023    /// `current_exe()` and why the existence check is here: `--lib` alone does
12024    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12025    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12026    pub(super) fn fake_aft_stub_path() -> PathBuf {
12027        let mut path = std::env::current_exe().expect("current_exe available in tests");
12028        path.pop();
12029        path.pop();
12030        path.push(if cfg!(windows) {
12031            "fake-aft-stub.exe"
12032        } else {
12033            "fake-aft-stub"
12034        });
12035        assert!(
12036            path.exists(),
12037            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12038             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12039            path.display()
12040        );
12041        path
12042    }
12043
12044    #[test]
12045    fn reserved_never_spawned_refuses_every_hello() {
12046        // The canary hole: a reserved id whose module has never spawned had NO
12047        // gate entry and admitted anyone -- the reservation protected the nonce
12048        // holder, not the NAME. Now the entry is present with no legitimate
12049        // holder and refuses all comers.
12050        let supervisor = SupervisorHandle::default();
12051        supervisor.apply_identity_configuration(&ModuleSpec {
12052            module_id: "never-spawned".to_string(),
12053            program: PathBuf::from("/usr/bin/false"),
12054            args: Vec::new(),
12055            env: Vec::new(),
12056            reserved: true,
12057            reserved_prefixes: Vec::new(),
12058            protocol: ModuleProtocol::Subc,
12059            overlap: Default::default(),
12060        });
12061        assert!(
12062            supervisor
12063                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12064                .is_err(),
12065            "forged nonce must refuse on a reserved never-spawned id"
12066        );
12067        assert!(
12068            supervisor
12069                .reserved_hello_verification("never-spawned", None)
12070                .is_err(),
12071            "absent nonce must refuse on a reserved never-spawned id"
12072        );
12073        // And a real spawn nonce minted later admits exactly that nonce.
12074        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12075        supervisor.apply_identity_configuration(&ModuleSpec {
12076            module_id: "never-spawned".to_string(),
12077            program: PathBuf::from("/usr/bin/false"),
12078            args: Vec::new(),
12079            env: Vec::new(),
12080            reserved: true,
12081            reserved_prefixes: Vec::new(),
12082            protocol: ModuleProtocol::Subc,
12083            overlap: Default::default(),
12084        });
12085        assert!(supervisor
12086            .reserved_hello_verification("never-spawned", Some("minted"))
12087            .is_ok());
12088        assert!(supervisor
12089            .reserved_hello_verification("never-spawned", Some("forged"))
12090            .is_err());
12091    }
12092
12093    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12094    /// happened, which is what "spent budget" looks like to every reader.
12095    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12096        let now = ClockInstant::now();
12097        for _ in 0..count {
12098            state.crash_restarts.push_back(now);
12099        }
12100    }
12101
12102    /// Age the oldest recorded restart out of `window`, standing in for the hours
12103    /// that would otherwise have to pass. Injecting the instant is the point: a
12104    /// test that slept a real window would take ten minutes and still prove less.
12105    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12106        let aged = state
12107            .crash_restarts
12108            .front()
12109            .expect("a crash restart must be recorded before it can be aged")
12110            .checked_sub(window + Duration::from_secs(1))
12111            .expect("the test clock is far enough from its origin to age an instant");
12112        state.crash_restarts[0] = aged;
12113    }
12114
12115    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12116        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12117        seed_crash_restarts(&mut state, count);
12118        state
12119    }
12120
12121    #[test]
12122    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12123        let policy = RestartPolicy::new(3, Duration::ZERO);
12124        let now = ClockInstant::now();
12125        assert!(daemon_will_restart(
12126            &mut snapshot_with_restarts(true, 2),
12127            &policy,
12128            now
12129        ));
12130        assert!(!daemon_will_restart(
12131            &mut snapshot_with_restarts(true, 3),
12132            &policy,
12133            now
12134        ));
12135        assert!(!daemon_will_restart(
12136            &mut snapshot_with_restarts(false, 0),
12137            &policy,
12138            now
12139        ));
12140    }
12141
12142    #[test]
12143    fn crash_restart_backoff_escalates_with_in_window_count() {
12144        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12145            .with_max_backoff(Duration::from_secs(30));
12146        let now = ClockInstant::now();
12147        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12148        let schedules = (0..4)
12149            .map(|_| {
12150                state
12151                    .next_crash_restart(&policy, now)
12152                    .expect("the test policy allows four crash restarts")
12153            })
12154            .collect::<Vec<_>>();
12155
12156        assert_eq!(
12157            schedules
12158                .iter()
12159                .map(|schedule| schedule.restart_in_window)
12160                .collect::<Vec<_>>(),
12161            vec![0, 1, 2, 3]
12162        );
12163        assert_eq!(
12164            schedules
12165                .iter()
12166                .map(|schedule| schedule.delay)
12167                .collect::<Vec<_>>(),
12168            vec![
12169                Duration::from_millis(100),
12170                Duration::from_secs(1),
12171                Duration::from_secs(10),
12172                Duration::from_secs(30),
12173            ]
12174        );
12175    }
12176
12177    #[test]
12178    fn crash_restart_backoff_resets_after_ring_clear() {
12179        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12180        let now = ClockInstant::now();
12181        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12182        assert_eq!(
12183            state.next_crash_restart(&policy, now).unwrap().delay,
12184            Duration::from_millis(100)
12185        );
12186        assert_eq!(
12187            state.next_crash_restart(&policy, now).unwrap().delay,
12188            Duration::from_secs(1)
12189        );
12190
12191        state.clear_crash_restarts();
12192        let schedule = state
12193            .next_crash_restart(&policy, now)
12194            .expect("a cleared ring must allow another restart");
12195        assert_eq!(schedule.restart_in_window, 0);
12196        assert_eq!(schedule.delay, Duration::from_millis(100));
12197    }
12198
12199    #[test]
12200    fn crash_restart_backoff_ignores_aged_restarts() {
12201        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12202        let now = ClockInstant::now();
12203        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12204        state
12205            .next_crash_restart(&policy, now)
12206            .expect("the first restart is allowed");
12207        state
12208            .next_crash_restart(&policy, now)
12209            .expect("the second restart is allowed");
12210        state.crash_restarts[0] = now
12211            .checked_sub(policy.window + Duration::from_secs(1))
12212            .expect("the fake clock can age a restart past the window");
12213
12214        let schedule = state
12215            .next_crash_restart(&policy, now)
12216            .expect("an aged restart must release its slot");
12217        assert_eq!(schedule.restart_in_window, 1);
12218        assert_eq!(schedule.delay, Duration::from_secs(1));
12219        assert_eq!(state.crash_restarts.len(), 2);
12220    }
12221
12222    /// The budget is a rate: the same three spent restarts refuse a respawn
12223    /// while they are recent and allow one once they have aged past the window.
12224    /// Nothing about the module changed in between, which is the whole point.
12225    #[test]
12226    fn a_budget_spent_before_the_window_no_longer_refuses() {
12227        let policy = RestartPolicy::new(3, Duration::ZERO);
12228        let mut state = snapshot_with_restarts(true, 3);
12229        let now = ClockInstant::now();
12230        assert!(!daemon_will_restart(&mut state, &policy, now));
12231
12232        assert!(daemon_will_restart(
12233            &mut state,
12234            &policy,
12235            now + policy.window + Duration::from_secs(1)
12236        ));
12237        assert!(
12238            state.crash_restarts.is_empty(),
12239            "reading the budget must drop the instants that left the window"
12240        );
12241    }
12242
12243    fn module_with_recovery_snapshot(
12244        state: ModuleState,
12245        enabled: bool,
12246        restart_count: u32,
12247    ) -> SupervisedModule {
12248        let registry = Arc::new(Registry::default());
12249        let supervisor =
12250            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12251        let runtime = supervisor.runtime_config();
12252        let spec = ModuleSpec {
12253            module_id: "recovery-snapshot".to_string(),
12254            program: fake_aft_stub_path(),
12255            args: Vec::new(),
12256            env: Vec::new(),
12257            reserved: false,
12258            reserved_prefixes: Vec::new(),
12259            protocol: ModuleProtocol::Subc,
12260            overlap: Default::default(),
12261        };
12262        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12263        seed_crash_restarts(&mut snapshot, restart_count);
12264        // These tests read synthetic snapshots. A real child and monitor would
12265        // race those reads by replacing the requested state during startup.
12266        let (commands, _rx) = mpsc::channel(4);
12267        SupervisedModule {
12268            inner: Arc::new(SupervisedModuleInner {
12269                module_id: spec.module_id.clone(),
12270                registry,
12271                snapshot: Arc::new(Mutex::new(snapshot)),
12272                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12273                    spec,
12274                    health: runtime.health,
12275                })),
12276                stderr_ring: runtime.stderr_ring,
12277                terminal_ring: runtime.terminal_ring,
12278                commands,
12279                monitor: Mutex::new(None),
12280                restart_policy: runtime.restart_policy,
12281                effective_drain_timeout: runtime.effective_drain_timeout,
12282                provenance_probe: supervisor.provenance_probe.clone(),
12283            }),
12284        }
12285    }
12286
12287    #[cfg(target_os = "linux")]
12288    #[tokio::test]
12289    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12290        let supervisor =
12291            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12292                .with_cgroup_placement(None);
12293        let result = supervisor.spawn(ModuleSpec {
12294            module_id: "no-cgroup-placement".to_string(),
12295            program: fake_aft_stub_path(),
12296            args: Vec::new(),
12297            env: Vec::new(),
12298            reserved: false,
12299            reserved_prefixes: Vec::new(),
12300            protocol: ModuleProtocol::Subc,
12301            overlap: Default::default(),
12302        });
12303
12304        assert!(
12305            result.is_ok(),
12306            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12307        );
12308    }
12309
12310    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12311    async fn undecided_snapshot_uses_shared_restart_predicate() {
12312        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12313            .will_recover_after_connection_loss()
12314            .unwrap());
12315        assert!(
12316            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12317                .will_recover_after_connection_loss()
12318                .unwrap()
12319        );
12320    }
12321
12322    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12323    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12324        assert!(
12325            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12326                .will_recover_after_connection_loss()
12327                .unwrap()
12328        );
12329    }
12330
12331    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12332    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12333        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12334            .will_recover_after_connection_loss()
12335            .unwrap());
12336        assert!(
12337            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12338                .will_recover_after_connection_loss()
12339                .unwrap()
12340        );
12341    }
12342
12343    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12344    async fn warming_snapshot_is_limited_to_startup_phases() {
12345        for state in [
12346            ModuleState::Starting,
12347            ModuleState::Running,
12348            ModuleState::Restarting,
12349        ] {
12350            assert!(
12351                module_with_recovery_snapshot(state, true, 0)
12352                    .is_warming()
12353                    .unwrap(),
12354                "{state:?} should be warming"
12355            );
12356        }
12357        for state in [
12358            ModuleState::Unresponsive,
12359            ModuleState::Draining,
12360            ModuleState::Stopped,
12361            ModuleState::Failed,
12362            ModuleState::Disabled,
12363        ] {
12364            assert!(
12365                !module_with_recovery_snapshot(state, true, 0)
12366                    .is_warming()
12367                    .unwrap(),
12368                "{state:?} should not be warming"
12369            );
12370        }
12371    }
12372
12373    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12374    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12375        let registry = Arc::new(Registry::default());
12376        let supervisor =
12377            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12378        let module = supervisor
12379            .spawn(ModuleSpec {
12380                module_id: "terminal-history".to_string(),
12381                program: fake_aft_stub_path(),
12382                args: Vec::new(),
12383                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12384                reserved: false,
12385                reserved_prefixes: Vec::new(),
12386                protocol: ModuleProtocol::Subc,
12387                overlap: Default::default(),
12388            })
12389            .unwrap();
12390
12391        let deadline = Instant::now() + Duration::from_secs(5);
12392        loop {
12393            let history = module.terminal_history();
12394            if history.entries.len() == 2 {
12395                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12396                assert_eq!(history.dropped, 0);
12397                assert_eq!(
12398                    history
12399                        .entries
12400                        .iter()
12401                        .map(|entry| entry.exit_code)
12402                        .collect::<Vec<_>>(),
12403                    vec![Some(23), Some(23)]
12404                );
12405                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12406                return;
12407            }
12408            assert!(
12409                Instant::now() < deadline,
12410                "module did not retain two terminal exits: {history:?}"
12411            );
12412            sleep(Duration::from_millis(10)).await;
12413        }
12414    }
12415
12416    /// A disable issued while a crash respawn is still backing off must preempt
12417    /// that respawn: the operator's stop wins, the disable must not queue behind
12418    /// the backoff, and the module must never come back up afterwards.
12419    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12420    async fn disable_during_crash_backoff_cancels_pending_respawn() {
12421        let backoff = Duration::from_secs(2);
12422        let supervisor = Supervisor::new_for_test(
12423            Arc::new(Registry::default()),
12424            RestartPolicy::new(10, backoff),
12425        );
12426        let module = supervisor
12427            .spawn(ModuleSpec {
12428                module_id: "disable-during-backoff".to_string(),
12429                program: fake_aft_stub_path(),
12430                args: Vec::new(),
12431                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12432                reserved: false,
12433                reserved_prefixes: Vec::new(),
12434                protocol: ModuleProtocol::Subc,
12435                overlap: Default::default(),
12436            })
12437            .unwrap();
12438
12439        // Wait for the first crash to put the module into its backoff window.
12440        let deadline = Instant::now() + Duration::from_secs(5);
12441        loop {
12442            if module.status().unwrap().state == ModuleState::Restarting {
12443                break;
12444            }
12445            assert!(
12446                Instant::now() < deadline,
12447                "module never entered the crash backoff"
12448            );
12449            sleep(Duration::from_millis(10)).await;
12450        }
12451
12452        let started = Instant::now();
12453        module.set_enabled(false).await.unwrap();
12454        let waited = started.elapsed();
12455
12456        assert!(
12457            waited < backoff / 2,
12458            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12459        );
12460        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12461
12462        // Outlast the backoff: the respawn it was counting down to must never run.
12463        sleep(backoff + Duration::from_millis(500)).await;
12464        let status = module.status().unwrap();
12465        assert_eq!(status.state, ModuleState::Disabled);
12466        assert_eq!(
12467            status.spawn_generation, 1,
12468            "module respawned after the operator disabled it"
12469        );
12470    }
12471
12472    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12473    /// the shape of nats-server, the program this rule exists for.
12474    #[cfg(unix)]
12475    fn protocol_none_sigterm_exits_clean_spec(
12476        module_id: &str,
12477        dir: &std::path::Path,
12478    ) -> (ModuleSpec, PathBuf, PathBuf) {
12479        let ready = dir.join("ready");
12480        let marker = dir.join("sigterm");
12481        let spec = ModuleSpec {
12482            module_id: module_id.to_string(),
12483            program: fake_aft_stub_path(),
12484            args: Vec::new(),
12485            env: vec![
12486                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12487                (
12488                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12489                    marker.display().to_string(),
12490                ),
12491                (
12492                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12493                    ready.display().to_string(),
12494                ),
12495            ],
12496            reserved: false,
12497            reserved_prefixes: Vec::new(),
12498            protocol: ModuleProtocol::None,
12499            overlap: Default::default(),
12500        };
12501        (spec, ready, marker)
12502    }
12503
12504    /// Wait for a file the child writes, so a signal is never sent before the
12505    /// child's SIGTERM handler is installed (the default disposition would
12506    /// kill it by signal and the exit would not be clean).
12507    #[cfg(unix)]
12508    async fn wait_for_file(path: &std::path::Path) {
12509        let deadline = Instant::now() + Duration::from_secs(10);
12510        while !path.exists() {
12511            assert!(
12512                Instant::now() < deadline,
12513                "{} never appeared",
12514                path.display()
12515            );
12516            sleep(Duration::from_millis(10)).await;
12517        }
12518    }
12519
12520    /// A protocol-none module that exits 0 because something OUTSIDE the
12521    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12522    /// the crash-path disposition rather than `stopped`.
12523    #[cfg(unix)]
12524    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12525    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12526        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12527        let (spec, ready, marker) =
12528            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12529        let supervisor = Supervisor::new_for_test(
12530            Arc::new(Registry::default()),
12531            RestartPolicy::new(3, Duration::ZERO),
12532        );
12533        let module = supervisor.spawn(spec).unwrap();
12534        wait_for_file(&ready).await;
12535        // The ready file proves the child installed its SIGTERM handler, not
12536        // that the supervisor has processed the privacy trampoline's exec
12537        // acknowledgement. On macOS status withholds the pid until then.
12538        let deadline = Instant::now() + Duration::from_secs(10);
12539        let first_pid = loop {
12540            let status = module.status().unwrap();
12541            if status.state == ModuleState::Running {
12542                if let Some(pid) = status.pid {
12543                    break pid;
12544                }
12545            }
12546            assert!(
12547                Instant::now() < deadline,
12548                "a running module must report its pid after exec confirmation: {status:?}"
12549            );
12550            sleep(Duration::from_millis(10)).await;
12551        };
12552
12553        rustix::process::kill_process(
12554            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12555            rustix::process::Signal::TERM,
12556        )
12557        .unwrap();
12558
12559        let deadline = Instant::now() + Duration::from_secs(10);
12560        let respawned = loop {
12561            let status = module.status().unwrap();
12562            if status.state == ModuleState::Running
12563                && status.pid.is_some_and(|pid| pid != first_pid)
12564            {
12565                break status;
12566            }
12567            assert!(
12568                Instant::now() < deadline,
12569                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12570            );
12571            sleep(Duration::from_millis(10)).await;
12572        };
12573        assert_eq!(respawned.spawn_generation, 2);
12574        assert!(
12575            marker.exists(),
12576            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12577        );
12578
12579        let history = module.terminal_history();
12580        assert_eq!(history.entries.len(), 1, "{history:?}");
12581        let entry = &history.entries[0];
12582        assert_eq!(entry.exit_code, Some(0));
12583        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12584        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12585
12586        module.stop().await.unwrap();
12587    }
12588
12589    /// Repeated unrequested clean exits of a protocol-none module spend the
12590    /// restart budget exactly as crashes do, and the module ends `failed` with
12591    /// the budget named.
12592    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12593    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12594        let supervisor = Supervisor::new_for_test(
12595            Arc::new(Registry::default()),
12596            RestartPolicy::new(1, Duration::ZERO),
12597        );
12598        let module = supervisor
12599            .spawn(ModuleSpec {
12600                module_id: "none-clean-exit-budget".to_string(),
12601                program: fake_aft_stub_path(),
12602                args: Vec::new(),
12603                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12604                reserved: false,
12605                reserved_prefixes: Vec::new(),
12606                protocol: ModuleProtocol::None,
12607                overlap: Default::default(),
12608            })
12609            .unwrap();
12610
12611        // Failed follows the terminal write; this deadline only bounds a hang,
12612        // not an assumed duration for the two launches or their exit recording.
12613        let deadline = Instant::now() + Duration::from_secs(10);
12614        loop {
12615            let status = module.status().unwrap();
12616            if status.state == ModuleState::Failed {
12617                break;
12618            }
12619            assert!(
12620                Instant::now() < deadline,
12621                "module never exhausted its budget: {status:?} {:?}",
12622                module.terminal_history()
12623            );
12624            sleep(Duration::from_millis(10)).await;
12625        }
12626        let history = module.terminal_history();
12627        assert_eq!(
12628            history
12629                .entries
12630                .iter()
12631                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12632                .collect::<Vec<_>>(),
12633            vec![
12634                (Some(0), TerminalDisposition::Restarting),
12635                (Some(0), TerminalDisposition::Failed),
12636            ]
12637        );
12638        let detail = history.entries[1]
12639            .disposition_detail
12640            .as_deref()
12641            .expect("a budget failure names the budget");
12642        assert!(detail.contains("max_restarts=1"), "{detail}");
12643        assert_eq!(module.status().unwrap().spawn_generation, 2);
12644    }
12645
12646    #[test]
12647    fn restart_budget_failure_is_published_after_its_terminal_record() {
12648        use std::io::{Read, Write};
12649        let supervisor = Supervisor::new_for_test(
12650            Arc::new(Registry::default()),
12651            RestartPolicy::new(0, Duration::ZERO),
12652        );
12653        let runtime = supervisor.runtime_config();
12654        let spec = ModuleSpec {
12655            protocol: ModuleProtocol::None,
12656            ..windowed_crash_spec("budget-publication-order")
12657        };
12658        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12659            ModuleState::Running,
12660            true,
12661        )));
12662        lock_snapshot(&snapshot).unwrap().process_alive = true;
12663        let reaped_snapshot = snapshot.clone();
12664        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12665        let history = runtime.terminal_ring.clone();
12666        let reap = std::thread::spawn(move || {
12667            tokio::runtime::Builder::new_current_thread()
12668                .enable_all()
12669                .build()
12670                .unwrap()
12671                .block_on(on_child_exit(
12672                    &spec,
12673                    runtime.restart_policy,
12674                    &supervisor.registry,
12675                    &reaped_snapshot,
12676                    &runtime.terminal_ring,
12677                    &runtime.spawn_events,
12678                    &runtime.child_roster,
12679                    ExitReport {
12680                        kind: ExitKind::Clean,
12681                        code: Some(0),
12682                        signal: None,
12683                        at_ms: 1,
12684                    },
12685                    true,
12686                ))
12687        });
12688        let mut reached = [0];
12689        barrier.read_exact(&mut reached).unwrap();
12690        assert_eq!(&reached, b"R");
12691        let (before_state, before_alive) = {
12692            let state = lock_snapshot(&snapshot).unwrap();
12693            (state.state, state.process_alive)
12694        };
12695        let before_history = history.lock().unwrap().snapshot();
12696        barrier.write_all(b"X").unwrap();
12697        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12698        assert!(before_history.entries.is_empty());
12699        assert_ne!(
12700            before_state,
12701            ModuleState::Failed,
12702            "Failed was visible before its terminal record could be written"
12703        );
12704        assert!(before_alive);
12705        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12706        let history = history.lock().unwrap().snapshot();
12707        assert_eq!(history.entries.len(), 1);
12708        assert_eq!(history.entries[0].exit_code, Some(0));
12709        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12710    }
12711
12712    #[test]
12713    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12714        use std::io::{Read, Write};
12715
12716        let supervisor = Supervisor::new_for_test(
12717            Arc::new(Registry::default()),
12718            RestartPolicy::new(3, Duration::ZERO),
12719        );
12720        let mut runtime = supervisor.runtime_config();
12721        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12722        let snapshot = module.inner.snapshot.clone();
12723        lock_snapshot(&snapshot).unwrap().process_alive = true;
12724        runtime.terminal_ring = module.inner.terminal_ring.clone();
12725        let spec = windowed_crash_spec(module.module_id());
12726        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12727        let reaped_snapshot = snapshot.clone();
12728        let reap = std::thread::spawn(move || {
12729            tokio::runtime::Builder::new_current_thread()
12730                .enable_all()
12731                .build()
12732                .unwrap()
12733                .block_on(on_child_exit(
12734                    &spec,
12735                    runtime.restart_policy,
12736                    &supervisor.registry,
12737                    &reaped_snapshot,
12738                    &runtime.terminal_ring,
12739                    &runtime.spawn_events,
12740                    &runtime.child_roster,
12741                    ExitReport {
12742                        kind: ExitKind::Crash,
12743                        code: Some(2),
12744                        signal: None,
12745                        at_ms: 1,
12746                    },
12747                    true,
12748                ))
12749        });
12750
12751        // Hold the crash handler between its budget decision and publication.
12752        // Connection cleanup must still see the recovery that this crash chose,
12753        // even when that recovery consumes the last available restart slot.
12754        let mut reached = [0];
12755        barrier.read_exact(&mut reached).unwrap();
12756        assert_eq!(&reached, b"R");
12757        let before_status = module.status().unwrap();
12758        let before_history = module.terminal_history();
12759        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12760        barrier.write_all(b"X").unwrap();
12761        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12762
12763        assert_eq!(before_status.state, ModuleState::Running);
12764        assert!(before_status.process_alive);
12765        assert_eq!(before_status.restart_count, 3);
12766        assert!(before_history.entries.is_empty());
12767        assert!(
12768            before_recovers,
12769            "connection cleanup would report a terminal close for a scheduled crash restart"
12770        );
12771        let restarting = module.status().unwrap();
12772        assert_eq!(restarting.state, ModuleState::Restarting);
12773        assert_eq!(restarting.restart_count, 3);
12774        assert!(module.will_recover_after_connection_loss().unwrap());
12775        let history = module.terminal_history();
12776        assert_eq!(history.entries.len(), 1);
12777        assert_eq!(
12778            history.entries[0].disposition,
12779            TerminalDisposition::Restarting
12780        );
12781
12782        // Once the replacement is running, its next crash must be judged
12783        // against the exhausted budget rather than the previous recovery.
12784        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12785            state.state = ModuleState::Running;
12786            state.respawn_pending = false;
12787        })
12788        .unwrap();
12789        assert!(!module.will_recover_after_connection_loss().unwrap());
12790        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12791            state.enabled = false;
12792            state.respawn_pending = true;
12793        })
12794        .unwrap();
12795        assert!(!module.will_recover_after_connection_loss().unwrap());
12796    }
12797
12798    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12799        use std::net::{TcpListener, TcpStream};
12800        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12801        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12802        let (barrier, _) = listener.accept().unwrap();
12803        for stream in [&writer, &barrier] {
12804            stream
12805                .set_read_timeout(Some(Duration::from_secs(10)))
12806                .unwrap();
12807            stream
12808                .set_write_timeout(Some(Duration::from_secs(10)))
12809                .unwrap();
12810        }
12811        *events.1.lock().unwrap() = Some(writer);
12812        barrier
12813    }
12814
12815    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12816        use std::io::{Read, Write};
12817        let barrier = events.1.lock().unwrap().take();
12818        if let Some(mut barrier) = barrier {
12819            barrier.write_all(b"R").unwrap();
12820            let mut release = [0];
12821            barrier.read_exact(&mut release).unwrap();
12822            assert_eq!(&release, b"X");
12823        }
12824    }
12825
12826    #[test]
12827    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12828        use std::io::{Read, Write};
12829
12830        let supervisor = Supervisor::new_for_test(
12831            Arc::new(Registry::default()),
12832            RestartPolicy::new(0, Duration::ZERO),
12833        );
12834        let mut runtime = supervisor.runtime_config();
12835        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12836        let snapshot = module.inner.snapshot.clone();
12837        runtime.terminal_ring = module.inner.terminal_ring.clone();
12838        let spec = ModuleSpec {
12839            module_id: module.module_id().into(),
12840            program: fake_aft_stub_path(),
12841            protocol: ModuleProtocol::None,
12842            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12843            ..windowed_crash_spec(module.module_id())
12844        };
12845        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12846        let reap = std::thread::spawn(move || {
12847            tokio::runtime::Builder::new_current_thread()
12848                .enable_all()
12849                .build()
12850                .unwrap()
12851                .block_on(async {
12852                    let mut child =
12853                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12854                    health_restart_child(
12855                        &spec,
12856                        &runtime,
12857                        &supervisor.registry,
12858                        &supervisor.process_liveness,
12859                        &snapshot,
12860                        &mut child,
12861                        SupervisorHealthStatus::Failing,
12862                        Some("health probe did not answer"),
12863                        1,
12864                    )
12865                    .await
12866                    .unwrap();
12867                    assert!(child.is_none());
12868                });
12869        });
12870        // The loopback barrier acknowledges entry into terminal recording by
12871        // health_restart_child's drain, then waits for the test's release byte.
12872        // Status and history are observed while that writer is held, not after
12873        // an elapsed-time guess about when the health action has completed.
12874        let mut reached = [0];
12875        barrier.read_exact(&mut reached).unwrap();
12876        assert_eq!(&reached, b"R");
12877        let before_status = module.status().unwrap();
12878        let before_history = module.terminal_history();
12879        barrier.write_all(b"X").unwrap();
12880        reap.join().unwrap();
12881        assert!(before_history.entries.is_empty());
12882        assert_ne!(
12883            before_status.state,
12884            ModuleState::Failed,
12885            "Failed was visible before its terminal record could be written"
12886        );
12887        assert!(
12888            before_status.process_alive,
12889            "process death was visible before its terminal record could be written"
12890        );
12891        let failed = module.status().unwrap();
12892        assert_eq!(failed.state, ModuleState::Failed);
12893        assert!(!failed.process_alive);
12894        let history = module.terminal_history();
12895        assert_eq!(history.entries.len(), 1);
12896        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12897        assert_eq!(
12898            history.entries[0].disposition_detail.as_deref(),
12899            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12900        );
12901    }
12902
12903    #[test]
12904    fn observed_exits_publish_process_death_after_terminal_history() {
12905        use std::io::{Read, Write};
12906        for (kind, enabled, shutdown, wait_error, expected_state) in [
12907            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12908            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12909            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12910            (
12911                ExitKind::DeliberateSeverance,
12912                true,
12913                false,
12914                false,
12915                ModuleState::Restarting,
12916            ),
12917            #[cfg(unix)]
12918            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12919            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12920        ] {
12921            let supervisor = Supervisor::new_for_test(
12922                Arc::new(Registry::default()),
12923                RestartPolicy::new(1, Duration::ZERO),
12924            );
12925            let runtime = supervisor.runtime_config();
12926            #[cfg(unix)]
12927            if shutdown {
12928                runtime.child_roster.close();
12929            }
12930            #[cfg(not(unix))]
12931            let _ = shutdown;
12932            let spec = windowed_crash_spec("exit-publication-order");
12933            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12934                ModuleState::Running,
12935                enabled,
12936            )));
12937            lock_snapshot(&snapshot).unwrap().process_alive = true;
12938            let reaped_snapshot = snapshot.clone();
12939            let history = runtime.terminal_ring.clone();
12940            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12941            let reap = std::thread::spawn(move || {
12942                if wait_error {
12943                    super::fail_after_wait_error(
12944                        &spec.module_id,
12945                        &reaped_snapshot,
12946                        &runtime.terminal_ring,
12947                        &runtime.spawn_events,
12948                        "wait failed".into(),
12949                        &runtime.failure_context(true),
12950                    );
12951                } else {
12952                    tokio::runtime::Builder::new_current_thread()
12953                        .enable_all()
12954                        .build()
12955                        .unwrap()
12956                        .block_on(on_child_exit(
12957                            &spec,
12958                            runtime.restart_policy,
12959                            &supervisor.registry,
12960                            &reaped_snapshot,
12961                            &runtime.terminal_ring,
12962                            &runtime.spawn_events,
12963                            &runtime.child_roster,
12964                            ExitReport {
12965                                kind,
12966                                code: Some(0),
12967                                signal: None,
12968                                at_ms: 1,
12969                            },
12970                            true,
12971                        ));
12972                }
12973            });
12974            let mut reached = [0];
12975            barrier.read_exact(&mut reached).unwrap();
12976            assert_eq!(&reached, b"R");
12977            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12978            let before_history = history.lock().unwrap().snapshot();
12979            barrier.write_all(b"X").unwrap();
12980            reap.join().unwrap();
12981            assert!(
12982                before_alive,
12983                "{kind:?}: death was published before recording"
12984            );
12985            assert!(before_history.entries.is_empty());
12986            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12987            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12988            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12989        }
12990    }
12991
12992    #[tokio::test]
12993    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12994        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12995        let path = dir.join("terminals.jsonl");
12996        std::fs::create_dir(&path).unwrap();
12997        let supervisor = Supervisor::new_for_test(
12998            Arc::new(Registry::default()),
12999            RestartPolicy::new(0, Duration::ZERO),
13000        )
13001        .with_terminal_journal(path, "health-budget-journal-failure".into());
13002        let runtime = supervisor.runtime_config();
13003        let spec = ModuleSpec {
13004            program: fake_aft_stub_path(),
13005            protocol: ModuleProtocol::None,
13006            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13007            ..windowed_crash_spec("health-budget-journal-failure")
13008        };
13009        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13010        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13011        health_restart_child(
13012            &spec,
13013            &runtime,
13014            &supervisor.registry,
13015            &supervisor.process_liveness,
13016            &snapshot,
13017            &mut child,
13018            SupervisorHealthStatus::Failing,
13019            None,
13020            1,
13021        )
13022        .await
13023        .unwrap();
13024        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13025        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13026        let history = runtime
13027            .terminal_ring
13028            .lock()
13029            .unwrap()
13030            .durable_history(&spec.module_id);
13031        assert!(history.journal_write_failures > 0);
13032        assert_eq!(history.entries.len(), 1);
13033        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13034    }
13035
13036    #[test]
13037    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13038        let supervisor = Supervisor::new_for_test(
13039            Arc::new(Registry::default()),
13040            RestartPolicy::new(0, Duration::ZERO),
13041        );
13042        let runtime = supervisor.runtime_config();
13043        let spec = ModuleSpec {
13044            program: fake_aft_stub_path(),
13045            protocol: ModuleProtocol::None,
13046            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13047            ..windowed_crash_spec("health-budget-recording-panic")
13048        };
13049        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13050        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13051            tokio::runtime::Builder::new_current_thread()
13052                .enable_all()
13053                .build()
13054                .unwrap()
13055                .block_on(async {
13056                    let mut child =
13057                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13058                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13059                    health_restart_child(
13060                        &spec,
13061                        &runtime,
13062                        &supervisor.registry,
13063                        &supervisor.process_liveness,
13064                        &snapshot,
13065                        &mut child,
13066                        SupervisorHealthStatus::Failing,
13067                        None,
13068                        1,
13069                    )
13070                    .await
13071                    .unwrap();
13072                });
13073        }));
13074        let panic = result.expect_err("recording must still unwind");
13075        assert_eq!(
13076            panic.downcast_ref::<String>().map(String::as_str),
13077            Some("spawn event sequence exhausted")
13078        );
13079        let state = lock_snapshot(&snapshot).unwrap();
13080        assert_eq!(state.state, ModuleState::Failed);
13081        assert!(!state.process_alive);
13082        assert!(state.last_exit.is_some());
13083    }
13084
13085    #[tokio::test]
13086    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13087        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13088        let path = dir.join("terminals.jsonl");
13089        std::fs::create_dir(&path).unwrap();
13090        let supervisor = Supervisor::new_for_test(
13091            Arc::new(Registry::default()),
13092            RestartPolicy::new(0, Duration::ZERO),
13093        )
13094        .with_terminal_journal(path, "budget-journal-failure".into());
13095        let runtime = supervisor.runtime_config();
13096        let spec = windowed_crash_spec("budget-journal-failure");
13097        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13098        assert!(matches!(
13099            on_child_exit(
13100                &spec,
13101                runtime.restart_policy,
13102                &supervisor.registry,
13103                &snapshot,
13104                &runtime.terminal_ring,
13105                &runtime.spawn_events,
13106                &runtime.child_roster,
13107                crash_exit_report(1),
13108                true,
13109            )
13110            .await,
13111            NextAction::Stop { .. }
13112        ));
13113        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13114        let history = runtime
13115            .terminal_ring
13116            .lock()
13117            .unwrap()
13118            .durable_history(&spec.module_id);
13119        assert!(history.journal_write_failures > 0);
13120        assert_eq!(history.entries.len(), 1);
13121        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13122    }
13123
13124    #[test]
13125    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13126        let supervisor = Supervisor::new_for_test(
13127            Arc::new(Registry::default()),
13128            RestartPolicy::new(0, Duration::ZERO),
13129        );
13130        let runtime = supervisor.runtime_config();
13131        let spec = windowed_crash_spec("budget-recording-panic");
13132        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13133        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13134        // Exhausting the event sequence makes emit_exited panic before the
13135        // terminal write, exercising publication on the recording unwind.
13136        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13137        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13138            tokio::runtime::Builder::new_current_thread()
13139                .enable_all()
13140                .build()
13141                .unwrap()
13142                .block_on(on_child_exit(
13143                    &spec,
13144                    runtime.restart_policy,
13145                    &supervisor.registry,
13146                    &snapshot,
13147                    &runtime.terminal_ring,
13148                    &runtime.spawn_events,
13149                    &runtime.child_roster,
13150                    crash_exit_report(1),
13151                    true,
13152                ))
13153        }));
13154        let panic = result.err().expect("recording must still unwind");
13155        assert_eq!(
13156            panic.downcast_ref::<String>().map(String::as_str),
13157            Some("spawn event sequence exhausted")
13158        );
13159        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13160    }
13161
13162    /// A stop the supervisor itself requests still stops a protocol-none
13163    /// module, even though the child answers the SIGTERM with exit 0.
13164    #[cfg(unix)]
13165    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13166    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13167        for disable in [false, true] {
13168            let label = if disable {
13169                "none-requested-disable"
13170            } else {
13171                "none-requested-stop"
13172            };
13173            let dir = cortexkit_test_support::ScratchDir::new(label);
13174            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13175            let supervisor = Supervisor::new_for_test(
13176                Arc::new(Registry::default()),
13177                RestartPolicy::new(3, Duration::ZERO),
13178            );
13179            let module = supervisor.spawn(spec).unwrap();
13180            wait_for_file(&ready).await;
13181
13182            if disable {
13183                module.set_enabled(false).await.unwrap();
13184            } else {
13185                module.stop().await.unwrap();
13186            }
13187            assert!(
13188                marker.exists(),
13189                "{label}: the child must have left through its SIGTERM handler with exit 0"
13190            );
13191
13192            // Long enough for a zero-backoff respawn to have happened if the
13193            // exit had been treated as a crash.
13194            sleep(Duration::from_millis(500)).await;
13195            let status = module.status().unwrap();
13196            let expected = if disable {
13197                ModuleState::Disabled
13198            } else {
13199                ModuleState::Stopped
13200            };
13201            assert_eq!(status.state, expected, "{label}");
13202            assert_eq!(
13203                status.spawn_generation, 1,
13204                "{label}: respawned after a requested stop"
13205            );
13206            let history = module.terminal_history();
13207            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13208            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13209            assert_ne!(
13210                history.entries[0].disposition,
13211                TerminalDisposition::Restarting,
13212                "{label}"
13213            );
13214        }
13215    }
13216
13217    /// A subc-wire module that exits 0 on its own is still a stop: the
13218    /// protocol-none rule must not reach it.
13219    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13220    async fn subc_wire_clean_exit_is_still_a_stop() {
13221        let supervisor = Supervisor::new_for_test(
13222            Arc::new(Registry::default()),
13223            RestartPolicy::new(3, Duration::ZERO),
13224        );
13225        let module = supervisor
13226            .spawn(ModuleSpec {
13227                module_id: "wire-clean-exit".to_string(),
13228                program: fake_aft_stub_path(),
13229                args: Vec::new(),
13230                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13231                reserved: false,
13232                reserved_prefixes: Vec::new(),
13233                protocol: ModuleProtocol::Subc,
13234                overlap: Default::default(),
13235            })
13236            .unwrap();
13237
13238        let deadline = Instant::now() + Duration::from_secs(10);
13239        while module.terminal_history().entries.is_empty() {
13240            assert!(Instant::now() < deadline, "module never exited");
13241            sleep(Duration::from_millis(10)).await;
13242        }
13243        // Long enough for a zero-backoff respawn to have happened.
13244        sleep(Duration::from_millis(500)).await;
13245        let status = module.status().unwrap();
13246        assert_eq!(status.state, ModuleState::Stopped);
13247        assert_eq!(status.spawn_generation, 1);
13248        let history = module.terminal_history();
13249        assert_eq!(history.entries.len(), 1, "{history:?}");
13250        assert_eq!(history.entries[0].exit_code, Some(0));
13251        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13252    }
13253
13254    #[cfg(unix)]
13255    #[tokio::test]
13256    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13257        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13258        let record = dir.join("live-children.json");
13259        let supervisor = Supervisor::new_for_test(
13260            Arc::new(Registry::default()),
13261            RestartPolicy::new(0, Duration::ZERO),
13262        );
13263        let mut runtime = supervisor.runtime_config();
13264        runtime.child_roster.record_to(record.clone());
13265        let gate = Arc::new(super::ReloadExitRecordGate::default());
13266        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13267        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13268        let spec = ModuleSpec {
13269            module_id: "reload-exit-roster".into(),
13270            program: fake_aft_stub_path(),
13271            args: Vec::new(),
13272            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13273            reserved: false,
13274            reserved_prefixes: Vec::new(),
13275            protocol: ModuleProtocol::Subc,
13276            overlap: Default::default(),
13277        };
13278        let mut child = None;
13279        let reload = super::finish_reload_child(
13280            &spec,
13281            &runtime,
13282            &supervisor.registry,
13283            &supervisor.process_liveness,
13284            &snapshot,
13285            &mut child,
13286        );
13287        tokio::pin!(reload);
13288        tokio::select! {
13289            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13290            _ = gate.reached.notified() => {}
13291        }
13292        assert!(runtime
13293            .terminal_ring
13294            .lock()
13295            .unwrap()
13296            .snapshot()
13297            .entries
13298            .is_empty());
13299        assert_eq!(
13300            crate::live_children::read_record(&record).unwrap().len(),
13301            1,
13302            "shutdown must still wait for the reaped child until its terminal record exists"
13303        );
13304        runtime.child_roster.close();
13305        gate.resume.notify_one();
13306        assert!(reload.await.is_err());
13307        assert!(crate::live_children::read_record(&record)
13308            .unwrap()
13309            .is_empty());
13310        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13311        assert_eq!(history.entries.len(), 1);
13312        assert_eq!(
13313            history.entries[0].disposition,
13314            TerminalDisposition::DaemonShutdown
13315        );
13316    }
13317
13318    /// Each restart-producing arm has its own state transition. Keeping their
13319    /// lifetime count assertions adjacent prevents a later new arm from silently
13320    /// spending budget without recording the historical restart.
13321    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13322    async fn every_restart_increment_path_advances_lifetime_count() {
13323        let supervisor = Supervisor::new_for_test(
13324            Arc::new(Registry::default()),
13325            RestartPolicy::new(1, Duration::ZERO),
13326        );
13327        let runtime = supervisor.runtime_config();
13328        let spec = ModuleSpec {
13329            module_id: "lifetime-increment-path".to_string(),
13330            program: PathBuf::from("/unused/lifetime-increment-path"),
13331            args: Vec::new(),
13332            env: Vec::new(),
13333            reserved: false,
13334            reserved_prefixes: Vec::new(),
13335            protocol: ModuleProtocol::Subc,
13336            overlap: Default::default(),
13337        };
13338
13339        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13340        assert!(matches!(
13341            on_child_exit(
13342                &spec,
13343                runtime.restart_policy,
13344                &supervisor.registry,
13345                &crash_snapshot,
13346                &runtime.terminal_ring,
13347                &runtime.spawn_events,
13348                &runtime.child_roster,
13349                ExitReport {
13350                    kind: ExitKind::Crash,
13351                    code: Some(1),
13352                    signal: None,
13353                    at_ms: 1,
13354                },
13355                true,
13356            )
13357            .await,
13358            NextAction::Restart { schedule: _ }
13359        ));
13360        let (crash_restarts, crash_lifetime) = {
13361            let state = lock_snapshot(&crash_snapshot).unwrap();
13362            (state.crash_restarts.len(), state.lifetime_restarts)
13363        };
13364        assert_eq!(crash_restarts, 1);
13365        assert_eq!(crash_lifetime, 1);
13366
13367        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13368        let mut health_child = None;
13369        assert!(matches!(
13370            health_restart_child(
13371                &spec,
13372                &runtime,
13373                &supervisor.registry,
13374                &supervisor.process_liveness,
13375                &health_snapshot,
13376                &mut health_child,
13377                SupervisorHealthStatus::Failing,
13378                None,
13379                2,
13380            )
13381            .await,
13382            Ok(())
13383        ));
13384        assert!(health_child.is_none());
13385        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13386        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13387        let (health_restarts, health_lifetime) = {
13388            let state = lock_snapshot(&health_snapshot).unwrap();
13389            (state.crash_restarts.len(), state.lifetime_restarts)
13390        };
13391        assert_eq!(health_restarts, 1);
13392        assert_eq!(health_lifetime, 1);
13393
13394        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13395        let mut reload_child = None;
13396        assert!(matches!(
13397            handle_reload_spawn_failure(
13398                &spec,
13399                &runtime,
13400                &supervisor.process_liveness,
13401                &reload_snapshot,
13402                &mut reload_child,
13403                "forced reload spawn failure".to_string(),
13404            )
13405            .await,
13406            Err(SuperviseError::ReloadFailed { .. })
13407        ));
13408        let (reload_restarts, reload_lifetime) = {
13409            let state = lock_snapshot(&reload_snapshot).unwrap();
13410            (state.crash_restarts.len(), state.lifetime_restarts)
13411        };
13412        assert_eq!(reload_restarts, 1);
13413        assert_eq!(reload_lifetime, 1);
13414    }
13415
13416    #[tokio::test]
13417    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13418        let supervisor = Supervisor::new_for_test(
13419            Arc::new(Registry::default()),
13420            RestartPolicy::new(3, Duration::ZERO),
13421        );
13422        let runtime = supervisor.runtime_config();
13423        let spec = ModuleSpec {
13424            module_id: "deliberately-severed".to_string(),
13425            program: PathBuf::from("/unused/deliberately-severed"),
13426            args: Vec::new(),
13427            env: Vec::new(),
13428            reserved: false,
13429            reserved_prefixes: Vec::new(),
13430            protocol: ModuleProtocol::Subc,
13431            overlap: Default::default(),
13432        };
13433        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13434        let process = ProcessIdentity {
13435            pid: 41,
13436            start_time: 101,
13437        };
13438        record_deliberate_severance(&snapshot, process).unwrap();
13439        let exit_report = apply_deliberate_severance_marker(
13440            &snapshot,
13441            Some(process),
13442            ExitReport {
13443                kind: ExitKind::Crash,
13444                code: Some(1),
13445                signal: None,
13446                at_ms: 1,
13447            },
13448        );
13449        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13450
13451        assert!(matches!(
13452            on_child_exit(
13453                &spec,
13454                runtime.restart_policy,
13455                &supervisor.registry,
13456                &snapshot,
13457                &runtime.terminal_ring,
13458                &runtime.spawn_events,
13459                &runtime.child_roster,
13460                exit_report,
13461                true,
13462            )
13463            .await,
13464            NextAction::Restart { schedule: _ }
13465        ));
13466        let state = lock_snapshot(&snapshot).unwrap();
13467        assert_eq!(state.lifetime_restarts, 1);
13468        assert_eq!(state.crash_restarts.len(), 0);
13469    }
13470
13471    #[tokio::test]
13472    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13473        let supervisor = Supervisor::new_for_test(
13474            Arc::new(Registry::default()),
13475            RestartPolicy::new(3, Duration::ZERO),
13476        );
13477        let runtime = supervisor.runtime_config();
13478        let spec = ModuleSpec {
13479            module_id: "genuine-crash".to_string(),
13480            program: PathBuf::from("/unused/genuine-crash"),
13481            args: Vec::new(),
13482            env: Vec::new(),
13483            reserved: false,
13484            reserved_prefixes: Vec::new(),
13485            protocol: ModuleProtocol::Subc,
13486            overlap: Default::default(),
13487        };
13488        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13489
13490        assert!(matches!(
13491            on_child_exit(
13492                &spec,
13493                runtime.restart_policy,
13494                &supervisor.registry,
13495                &snapshot,
13496                &runtime.terminal_ring,
13497                &runtime.spawn_events,
13498                &runtime.child_roster,
13499                ExitReport {
13500                    kind: ExitKind::Crash,
13501                    code: Some(1),
13502                    signal: None,
13503                    at_ms: 1,
13504                },
13505                true,
13506            )
13507            .await,
13508            NextAction::Restart { schedule: _ }
13509        ));
13510        let state = lock_snapshot(&snapshot).unwrap();
13511        assert_eq!(state.lifetime_restarts, 1);
13512        assert_eq!(state.crash_restarts.len(), 1);
13513    }
13514
13515    fn crash_exit_report(at_ms: u64) -> ExitReport {
13516        ExitReport {
13517            kind: ExitKind::Crash,
13518            code: Some(1),
13519            signal: None,
13520            at_ms,
13521        }
13522    }
13523
13524    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13525        ModuleSpec {
13526            module_id: module_id.to_string(),
13527            program: PathBuf::from("/unused").join(module_id),
13528            args: Vec::new(),
13529            env: Vec::new(),
13530            reserved: false,
13531            reserved_prefixes: Vec::new(),
13532            protocol: ModuleProtocol::Subc,
13533            overlap: Default::default(),
13534        }
13535    }
13536
13537    /// A real crash loop still stops. Three crashes with nothing aging out spend
13538    /// a budget of two and the third respawn is refused, and both surfaces an
13539    /// operator has -- the log line and the retained terminal record -- name the
13540    /// window rather than only the cap, because `max_restarts=2` alone is what
13541    /// this budget used to mean.
13542    #[tokio::test]
13543    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13544        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13545        let supervisor = Supervisor::new_for_test(
13546            Arc::new(Registry::default()),
13547            RestartPolicy::new(2, Duration::ZERO),
13548        );
13549        let runtime = supervisor.runtime_config();
13550        let spec = windowed_crash_spec("crash-loop-in-window");
13551        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13552
13553        for attempt in 1..=2 {
13554            assert!(
13555                matches!(
13556                    on_child_exit(
13557                        &spec,
13558                        runtime.restart_policy,
13559                        &supervisor.registry,
13560                        &snapshot,
13561                        &runtime.terminal_ring,
13562                        &runtime.spawn_events,
13563                        &runtime.child_roster,
13564                        crash_exit_report(attempt),
13565                        true,
13566                    )
13567                    .await,
13568                    NextAction::Restart { schedule: _ }
13569                ),
13570                "crash {attempt} is inside the budget and must respawn"
13571            );
13572        }
13573
13574        assert!(matches!(
13575            on_child_exit(
13576                &spec,
13577                runtime.restart_policy,
13578                &supervisor.registry,
13579                &snapshot,
13580                &runtime.terminal_ring,
13581                &runtime.spawn_events,
13582                &runtime.child_roster,
13583                crash_exit_report(3),
13584                true,
13585            )
13586            .await,
13587            NextAction::Stop { .. }
13588        ));
13589
13590        {
13591            let state = lock_snapshot(&snapshot).unwrap();
13592            assert_eq!(state.state, ModuleState::Failed);
13593            assert_eq!(state.crash_restarts.len(), 2);
13594            assert_eq!(state.lifetime_restarts, 2);
13595        }
13596
13597        let history = runtime
13598            .terminal_ring
13599            .lock()
13600            .expect("terminal ring is not poisoned")
13601            .snapshot();
13602        let last = history
13603            .entries
13604            .last()
13605            .expect("the refused crash is retained");
13606        assert_eq!(last.disposition, TerminalDisposition::Failed);
13607        assert_eq!(
13608            last.disposition_detail.as_deref(),
13609            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13610        );
13611
13612        let captured = crate::router::test_log::captured_logs(&logs);
13613        assert!(
13614            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13615            "the stop must be logged with its window: {captured}"
13616        );
13617    }
13618
13619    /// The rate, stated as a test: three crashes where the first has aged past
13620    /// the window are two crashes as far as the budget is concerned, so the
13621    /// third respawn is allowed and the ring holds only the two recent ones.
13622    ///
13623    /// This is the case a lifetime counter got wrong -- and the case the daemon
13624    /// now hits routinely, since a module exits non-zero every time its
13625    /// connection to the daemon drops.
13626    #[tokio::test]
13627    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13628        let supervisor = Supervisor::new_for_test(
13629            Arc::new(Registry::default()),
13630            RestartPolicy::new(2, Duration::ZERO),
13631        );
13632        let runtime = supervisor.runtime_config();
13633        let spec = windowed_crash_spec("crash-across-windows");
13634        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13635
13636        for attempt in 1..=2 {
13637            assert!(matches!(
13638                on_child_exit(
13639                    &spec,
13640                    runtime.restart_policy,
13641                    &supervisor.registry,
13642                    &snapshot,
13643                    &runtime.terminal_ring,
13644                    &runtime.spawn_events,
13645                    &runtime.child_roster,
13646                    crash_exit_report(attempt),
13647                    true,
13648                )
13649                .await,
13650                NextAction::Restart { schedule: _ }
13651            ));
13652        }
13653
13654        // The oldest crash moves out of the window; nothing else about the
13655        // module changes.
13656        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13657            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13658        })
13659        .unwrap();
13660
13661        assert!(
13662            matches!(
13663                on_child_exit(
13664                    &spec,
13665                    runtime.restart_policy,
13666                    &supervisor.registry,
13667                    &snapshot,
13668                    &runtime.terminal_ring,
13669                    &runtime.spawn_events,
13670                    &runtime.child_roster,
13671                    crash_exit_report(3),
13672                    true,
13673                )
13674                .await,
13675                NextAction::Restart { schedule: _ }
13676            ),
13677            "a crash older than the window must not hold a budget slot"
13678        );
13679
13680        let state = lock_snapshot(&snapshot).unwrap();
13681        assert_eq!(state.state, ModuleState::Restarting);
13682        assert_eq!(
13683            state.crash_restarts.len(),
13684            2,
13685            "the aged instant is dropped and the new one takes its place"
13686        );
13687        assert_eq!(
13688            state.lifetime_restarts, 3,
13689            "the ledger counts every restart, including the ones the window forgot"
13690        );
13691    }
13692
13693    /// An operator restart hands the budget back whole, and the ledger keeps
13694    /// counting. Those are different questions -- "how close is this module to
13695    /// being stopped" and "how many times has it been replaced" -- and the
13696    /// operator action answers only the first.
13697    #[tokio::test]
13698    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13699        let supervisor = Supervisor::new_for_test(
13700            Arc::new(Registry::default()),
13701            RestartPolicy::new(2, Duration::ZERO),
13702        );
13703        let runtime = supervisor.runtime_config();
13704        let spec = windowed_crash_spec("operator-cleared-budget");
13705        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13706
13707        for attempt in 1..=2 {
13708            assert!(matches!(
13709                on_child_exit(
13710                    &spec,
13711                    runtime.restart_policy,
13712                    &supervisor.registry,
13713                    &snapshot,
13714                    &runtime.terminal_ring,
13715                    &runtime.spawn_events,
13716                    &runtime.child_roster,
13717                    crash_exit_report(attempt),
13718                    true,
13719                )
13720                .await,
13721                NextAction::Restart { schedule: _ }
13722            ));
13723        }
13724
13725        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13726        {
13727            let state = lock_snapshot(&snapshot).unwrap();
13728            assert!(
13729                state.crash_restarts.is_empty(),
13730                "an operator restart returns the full budget"
13731            );
13732            assert_eq!(
13733                state.lifetime_restarts, 2,
13734                "clearing the budget must not unmake the crashes"
13735            );
13736        }
13737
13738        assert!(
13739            matches!(
13740                on_child_exit(
13741                    &spec,
13742                    runtime.restart_policy,
13743                    &supervisor.registry,
13744                    &snapshot,
13745                    &runtime.terminal_ring,
13746                    &runtime.spawn_events,
13747                    &runtime.child_roster,
13748                    crash_exit_report(3),
13749                    true,
13750                )
13751                .await,
13752                NextAction::Restart { schedule: _ }
13753            ),
13754            "the cleared budget must be spendable again"
13755        );
13756        let state = lock_snapshot(&snapshot).unwrap();
13757        assert_eq!(state.crash_restarts.len(), 1);
13758        assert_eq!(state.lifetime_restarts, 3);
13759    }
13760
13761    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13762    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13763        let severed = ProcessIdentity {
13764            pid: 41,
13765            start_time: 101,
13766        };
13767        let successor = ProcessIdentity {
13768            pid: 41,
13769            start_time: 202,
13770        };
13771        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13772        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13773            state.pid = Some(successor.pid);
13774            state.process_start_time = Some(successor.start_time);
13775        })
13776        .unwrap();
13777        assert!(!module.record_deliberate_severance(severed).unwrap());
13778
13779        let exit_report = apply_deliberate_severance_marker(
13780            &module.inner.snapshot,
13781            Some(successor),
13782            ExitReport {
13783                kind: ExitKind::Crash,
13784                code: Some(1),
13785                signal: None,
13786                at_ms: 1,
13787            },
13788        );
13789
13790        assert_eq!(exit_report.kind, ExitKind::Crash);
13791    }
13792
13793    #[tokio::test]
13794    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13795        let registry = Registry::default();
13796        let supervisor = Supervisor::new_for_test(
13797            Arc::new(Registry::default()),
13798            RestartPolicy::new(3, Duration::ZERO),
13799        );
13800        let runtime = supervisor.runtime_config();
13801        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13802        let spec = ModuleSpec {
13803            module_id: "drain-deliberate-severance".to_string(),
13804            program: fake_aft_stub_path(),
13805            args: Vec::new(),
13806            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13807            reserved: false,
13808            reserved_prefixes: Vec::new(),
13809            protocol: ModuleProtocol::Subc,
13810            overlap: Default::default(),
13811        };
13812        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13813        let process = ProcessIdentity {
13814            pid: 41,
13815            start_time: 101,
13816        };
13817        child.process_identity = Some(process);
13818        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13819            state.pid = Some(process.pid);
13820            state.process_start_time = Some(process.start_time);
13821        })
13822        .unwrap();
13823        record_deliberate_severance(&snapshot, process).unwrap();
13824
13825        drain_child_to_state(
13826            &spec.module_id,
13827            spec.protocol,
13828            // The child exits on its own; no signal may change the exit this
13829            // test classifies.
13830            StopNotice::SentOverConnection,
13831            &registry,
13832            None,
13833            &snapshot,
13834            &runtime.terminal_ring,
13835            &runtime.spawn_events,
13836            child,
13837            Duration::from_secs(1),
13838            ModuleState::Stopped,
13839            Some(false),
13840            &super::FailureContext::operator(),
13841        )
13842        .await
13843        .unwrap();
13844
13845        let state = lock_snapshot(&snapshot).unwrap();
13846        assert_eq!(
13847            state.last_exit.as_ref().map(|exit| exit.kind),
13848            Some(ExitKind::DeliberateSeverance)
13849        );
13850        assert_eq!(state.lifetime_restarts, 1);
13851        assert_eq!(state.crash_restarts.len(), 0);
13852        drop(state);
13853        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13854        assert_eq!(
13855            history.entries[0].exit_kind,
13856            subc_control::TerminalExitKind::DeliberateSeverance
13857        );
13858    }
13859
13860    #[tokio::test]
13861    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13862        let registry = Registry::default();
13863        let supervisor = Supervisor::new_for_test(
13864            Arc::new(Registry::default()),
13865            RestartPolicy::new(3, Duration::ZERO),
13866        );
13867        let runtime = supervisor.runtime_config();
13868        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13869        let spec = ModuleSpec {
13870            module_id: "ordinary-drain".to_string(),
13871            program: fake_aft_stub_path(),
13872            args: Vec::new(),
13873            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13874            reserved: false,
13875            reserved_prefixes: Vec::new(),
13876            protocol: ModuleProtocol::Subc,
13877            overlap: Default::default(),
13878        };
13879        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13880
13881        drain_child_to_state(
13882            &spec.module_id,
13883            spec.protocol,
13884            // The child exits on its own; no signal may change the exit this
13885            // test classifies.
13886            StopNotice::SentOverConnection,
13887            &registry,
13888            None,
13889            &snapshot,
13890            &runtime.terminal_ring,
13891            &runtime.spawn_events,
13892            child,
13893            Duration::from_secs(1),
13894            ModuleState::Stopped,
13895            Some(false),
13896            &super::FailureContext::operator(),
13897        )
13898        .await
13899        .unwrap();
13900
13901        let state = lock_snapshot(&snapshot).unwrap();
13902        assert_eq!(
13903            state.last_exit.as_ref().map(|exit| exit.kind),
13904            Some(ExitKind::Crash)
13905        );
13906        assert_eq!(state.lifetime_restarts, 0);
13907        assert_eq!(state.crash_restarts.len(), 0);
13908    }
13909
13910    #[test]
13911    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13912        // The server's generic fatal-routing branch only knows that the
13913        // connection failed; it does not know that the daemon deliberately
13914        // initiated a process-killing severance. Keep this seam explicit so a
13915        // future connection error path cannot silently reintroduce the stale
13916        // exemption that mislabels a later genuine crash.
13917        assert!(!include_str!("server.rs")
13918            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13919    }
13920
13921    /// The `route.closed` `drained` value must be the quiescence wait's own
13922    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13923    /// measurement at all and `false` is the one honest constant. This is the exact
13924    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13925    /// on every return path, including the one that used to return early via `?`
13926    /// with `route.closing` already sent and no `route.closed` ever following.
13927    #[test]
13928    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13929        assert!(drained_after_quiescence_wait(&Ok(true)));
13930        assert!(!drained_after_quiescence_wait(&Ok(false)));
13931        assert!(!drained_after_quiescence_wait(&Err(
13932            SuperviseError::StatePoisoned { module_id: None }
13933        )));
13934    }
13935
13936    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13937    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13938    /// already reaped out-of-band) still leaves a terminal record rather than none
13939    /// at all. Triggering the real `wait()` I/O error from an integration test would
13940    /// need a genuine already-reaped-child race, which is OS-specific and not
13941    /// something this suite attempts elsewhere; this test instead verifies the
13942    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13943    /// the call site itself is verified by inspection to sit in that exact arm.
13944    #[test]
13945    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13946        let ring = Arc::new(Mutex::new(TerminalRing::new(
13947            TerminalRingConfig::default(),
13948            0,
13949        )));
13950        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13951
13952        let snapshot = ring.lock().unwrap().snapshot();
13953        assert_eq!(snapshot.entries.len(), 1);
13954        let entry = &snapshot.entries[0];
13955        assert_eq!(entry.exit_code, None);
13956        assert_eq!(entry.exit_signal, None);
13957        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13958    }
13959
13960    #[test]
13961    fn wait_error_exit_path_preserves_spawn_event_density() {
13962        let feed = super::SpawnEventFeed::default();
13963        feed.configure_incarnation("wait-error-density".to_string());
13964        feed.emit_spawned("wait-error", 41, 1);
13965        let ring = Arc::new(Mutex::new(TerminalRing::new(
13966            TerminalRingConfig::default(),
13967            0,
13968        )));
13969
13970        record_wait_error_terminal("wait-error", &ring, &feed);
13971        feed.emit_spawned("after-wait-error", 42, 2);
13972
13973        let state = feed.0.lock().unwrap();
13974        let sequences = state
13975            .events
13976            .iter()
13977            .map(|event| event.cursor.seq)
13978            .collect::<Vec<_>>();
13979        assert_eq!(sequences, vec![1, 2, 3]);
13980        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13981        assert_eq!(state.events[1].exit_code, None);
13982        assert_eq!(state.events[1].exit_signal, None);
13983    }
13984
13985    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13986    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13987    /// not a clean exit it never actually observed.
13988    #[test]
13989    fn wait_error_exit_report_is_classified_as_a_crash() {
13990        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13991    }
13992}
13993
13994#[cfg(all(test, unix))]
13995mod restart_ordering_tests {
13996    use super::*;
13997    use std::io::Write;
13998    use tokio::io::AsyncReadExt;
13999
14000    #[test]
14001    #[ignore = "child process fixture"]
14002    fn serving_child() {
14003        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14004            return;
14005        };
14006        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14007        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14008        for stream in listener.incoming() {
14009            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14010        }
14011    }
14012
14013    async fn restart_before_publication(requests: usize) {
14014        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14015        let address_file = home.path().join("address");
14016        let spec = ModuleSpec {
14017            module_id: "restart-serving-child".into(),
14018            program: std::env::current_exe().unwrap(),
14019            args: vec![
14020                "--exact".into(),
14021                "supervise::restart_ordering_tests::serving_child".into(),
14022                "--ignored".into(),
14023                "--nocapture".into(),
14024            ],
14025            env: vec![(
14026                "RESTART_TEST_ADDRESS_FILE".into(),
14027                address_file.display().to_string(),
14028            )],
14029            reserved: false,
14030            reserved_prefixes: vec![],
14031            protocol: ModuleProtocol::None,
14032            overlap: Default::default(),
14033        };
14034        let mut supervisor =
14035            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14036        let (spawned_tx, spawned_rx) = oneshot::channel();
14037        let spawned_tx = Mutex::new(Some(spawned_tx));
14038        let (release_tx, release_rx) = std::sync::mpsc::channel();
14039        let release_rx = Mutex::new(release_rx);
14040        // Hold publication, not the child: it must answer a real request before
14041        // the restart enters the queue, even though status still has generation 0.
14042        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14043            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14044            release_rx
14045                .lock()
14046                .unwrap()
14047                .recv_timeout(Duration::from_secs(20))
14048                .unwrap();
14049        })));
14050        let registered = supervisor
14051            .register_configured_with_health(
14052                spec.clone(),
14053                true,
14054                HealthConfig::default(),
14055                None,
14056                RestartPolicy::new(3, Duration::ZERO),
14057                DEFAULT_REGISTER_TIMEOUT,
14058            )
14059            .unwrap();
14060        let module = registered.module.clone();
14061        let startup = tokio::task::spawn_blocking(move || {
14062            supervisor.start_registered_configured(registered, None, true)
14063        });
14064        spawned_rx.await.unwrap();
14065        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14066            let address = loop {
14067                if let Ok(address) = std::fs::read_to_string(&address_file) {
14068                    if !address.is_empty() {
14069                        break address;
14070                    }
14071                }
14072                tokio::time::sleep(Duration::from_millis(1)).await;
14073            };
14074            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14075            let mut reply = String::new();
14076            stream.read_to_string(&mut reply).await.unwrap();
14077            reply.trim().parse::<u32>().unwrap()
14078        })
14079        .await
14080        .expect("child must serve before generation publication");
14081        assert_eq!(
14082            lock_snapshot(&module.inner.snapshot)
14083                .unwrap()
14084                .spawn_generation,
14085            0
14086        );
14087        let mut replies = Vec::new();
14088        for _ in 0..requests {
14089            let mut restart = Box::pin(module.restart(Some(0)));
14090            // Poll the public API through queue admission, leaving only its ACK
14091            // pending. The monitor cannot consume the command until released.
14092            std::future::poll_fn(|cx| {
14093                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14094                std::task::Poll::Ready(())
14095            })
14096            .await;
14097            replies.push(restart);
14098        }
14099        release_tx.send(()).unwrap();
14100        startup.await.unwrap().unwrap();
14101        for reply in replies {
14102            reply.await.unwrap();
14103        }
14104        // Restart schedules its replacement after completing the handler. Wait
14105        // for that separate actor turn before placing the command barrier.
14106        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14107            loop {
14108                if lock_snapshot(&module.inner.snapshot)
14109                    .unwrap()
14110                    .spawn_generation
14111                    > 1
14112                {
14113                    break;
14114                }
14115                tokio::time::sleep(Duration::from_millis(1)).await;
14116            }
14117        })
14118        .await;
14119        // Commands are handled serially. The update-spec reply proves every
14120        // restart handler has finished before we inspect the snapshot; restart's
14121        // own ACK only acknowledges initiation, not replacement completion.
14122        module.update_spec_for_test(spec).await.unwrap();
14123        let (generation, pid) = {
14124            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14125            (state.spawn_generation, state.pid)
14126        };
14127        module.stop().await.unwrap();
14128        replacement.expect("restart must publish a replacement for the already-serving child");
14129        assert_ne!(
14130            pid,
14131            Some(old_pid),
14132            "a serving child predating the restart must be replaced"
14133        );
14134        assert_eq!(generation, 2, "queued restarts must share one replacement");
14135    }
14136
14137    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14138    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14139        restart_before_publication(1).await;
14140    }
14141
14142    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14143    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14144        restart_before_publication(2).await;
14145    }
14146
14147    #[tokio::test]
14148    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14149        let supervisor =
14150            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14151        let registered = supervisor
14152            .register_configured_with_health(
14153                ModuleSpec {
14154                    module_id: "queued-before-replacement".into(),
14155                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14156                    // macOS a module whose running image is not the configured
14157                    // program is refused once its exec is confirmed (see
14158                    // `record_write_stall_tests::sleeping_spec`).
14159                    program: "/bin/sleep".into(),
14160                    args: vec!["600".into()],
14161                    env: vec![],
14162                    reserved: false,
14163                    reserved_prefixes: vec![],
14164                    protocol: ModuleProtocol::None,
14165                    overlap: Default::default(),
14166                },
14167                true,
14168                HealthConfig::default(),
14169                None,
14170                RestartPolicy::default(),
14171                DEFAULT_REGISTER_TIMEOUT,
14172            )
14173            .unwrap();
14174        let RegisteredConfiguredModule {
14175            mut spec,
14176            mut runtime,
14177            module,
14178            mut commands,
14179        } = registered;
14180        let snapshot = &module.inner.snapshot;
14181        let mut restart = Box::pin(module.restart(Some(0)));
14182        std::future::poll_fn(|cx| {
14183            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14184            std::task::Poll::Ready(())
14185        })
14186        .await;
14187        // Drive the actor explicitly so a replacement is definitely spawned
14188        // after receipt, but before this queued command is dequeued.
14189        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14190        let replacement_pid = child.as_ref().unwrap().id();
14191        let command = commands.recv().await.unwrap();
14192        assert!(
14193            handle_supervisor_command(
14194                command,
14195                &mut spec,
14196                &mut runtime,
14197                &supervisor.registry,
14198                &supervisor.process_liveness,
14199                snapshot,
14200                &mut child,
14201                &mut commands,
14202                &mut VecDeque::new(),
14203            )
14204            .await
14205        );
14206        restart.await.unwrap();
14207        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14208        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14209        if let Some(mut child) = child {
14210            child.start_kill().unwrap();
14211            child.wait().await.unwrap();
14212        }
14213        assert_eq!(
14214            kept_pid, replacement_pid,
14215            "a newer replacement must survive"
14216        );
14217        assert!(
14218            !pending,
14219            "a satisfied restart must not schedule another spawn"
14220        );
14221    }
14222}
14223
14224#[cfg(all(test, unix))]
14225mod health_event_tests {
14226    use super::*;
14227    use cortexkit_test_support::ScratchDir;
14228    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14229
14230    struct Actor {
14231        module: SupervisedModule,
14232        registry: Arc<Registry>,
14233        forwarding: Arc<ForwardingTable>,
14234        spec: ModuleSpec,
14235        health: HealthConfig,
14236        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14237        roster: ChildRoster,
14238        _home: ScratchDir,
14239    }
14240
14241    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14242        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14243        // for an observed condition, rather than letting paused time auto-advance
14244        // deadlines or assuming a fixed number of yields finishes the work.
14245        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14246        let tick = Instant::now();
14247        loop {
14248            if let Some(value) = observe() {
14249                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14250                return value;
14251            }
14252            assert!(
14253                std::time::Instant::now() < deadline,
14254                "timed out waiting for {reason}"
14255            );
14256            tokio::task::yield_now().await;
14257        }
14258    }
14259
14260    impl Actor {
14261        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14262            Self::start_with_policy(
14263                protocol,
14264                cadence,
14265                DEFAULT_REGISTER_TIMEOUT,
14266                RestartPolicy::new(3, Duration::ZERO),
14267            )
14268            .await
14269        }
14270
14271        async fn start_with_policy(
14272            protocol: ModuleProtocol,
14273            cadence: Duration,
14274            register_timeout: Duration,
14275            policy: RestartPolicy,
14276        ) -> Self {
14277            let home = ScratchDir::new("health-events");
14278            let registry = Arc::new(Registry::default());
14279            let forwarding = Arc::new(ForwardingTable::default());
14280            let health = HealthConfig {
14281                cadence,
14282                deadline: Duration::from_secs(3600),
14283                ..HealthConfig::default()
14284            };
14285            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14286                .with_forwarding(forwarding.clone())
14287                .with_health_config(health.clone());
14288            let spec = ModuleSpec {
14289                module_id: "event-health-child".into(),
14290                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14291                // the privacy trampoline refuses a module whose running image
14292                // is not the configured program, and `start` below waits for
14293                // the process to be confirmed (`reported_pid`). The shell form
14294                // passed only while the image check ran before the shell had
14295                // replaced itself with `sleep` (see
14296                // `record_write_stall_tests::sleeping_spec`).
14297                program: PathBuf::from("/bin/sleep"),
14298                // The process stays alive but has no real bus peer. Registrations
14299                // below exercise the same registry writes as accepted HELLOs.
14300                args: vec!["600".into()],
14301                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14302                    .into_iter()
14303                    .map(|key| (key.into(), home.path().display().to_string()))
14304                    .collect(),
14305                reserved: false,
14306                reserved_prefixes: vec![],
14307                protocol,
14308                overlap: Default::default(),
14309            };
14310            let module = supervisor
14311                .supervise_configured_requested(
14312                    spec.clone(),
14313                    true,
14314                    health.clone(),
14315                    Some(0),
14316                    policy,
14317                    register_timeout,
14318                    None,
14319                )
14320                .unwrap();
14321            let (_, rx) = mpsc::channel(8);
14322            let actor = Self {
14323                module,
14324                registry,
14325                forwarding,
14326                spec,
14327                health,
14328                rx,
14329                roster: supervisor.child_roster.clone(),
14330                _home: home,
14331            };
14332            actor
14333                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14334                .await;
14335            actor
14336        }
14337
14338        async fn expire_registration_wait(&self, remaining: Duration) {
14339            tokio::time::advance(remaining).await;
14340            wait_for("registration timeout TERM grace", || {
14341                lock_snapshot(&self.module.inner.snapshot)
14342                    .unwrap()
14343                    .actor_select
14344                    .is_none()
14345                    .then_some(())
14346            })
14347            .await;
14348            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14349        }
14350
14351        fn turns(&self) -> u64 {
14352            lock_snapshot(&self.module.inner.snapshot)
14353                .unwrap()
14354                .actor_turns
14355        }
14356
14357        async fn wait_for_select(
14358            &self,
14359            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14360        ) -> ActorSelectCheckpoint {
14361            wait_for("actor select checkpoint", || {
14362                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14363                state.actor_select.filter(|checkpoint| {
14364                    checkpoint.generation == state.spawn_generation
14365                        && state.state == ModuleState::Running
14366                        && state.process_alive
14367                        && state.reported_pid().is_some()
14368                        && expected(checkpoint)
14369                })
14370            })
14371            .await
14372        }
14373
14374        fn hello(&mut self, connection: u64, health: bool) {
14375            let manifest =
14376                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14377                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14378                    .build();
14379            let connection = ConnectionId::new(connection);
14380            let (tx, rx) = mpsc::channel(8);
14381            self.rx = rx;
14382            self.forwarding
14383                .register_module_connection(
14384                    connection,
14385                    self.spec.module_id.clone(),
14386                    subc_protocol::PROTOCOL_VERSION,
14387                    Concurrency::ModuleManaged,
14388                    FrameSink::new(tx),
14389                )
14390                .unwrap();
14391            self.registry
14392                .register_with_control_ops(
14393                    manifest,
14394                    subc_protocol::PROTOCOL_VERSION,
14395                    connection,
14396                    if health {
14397                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14398                    } else {
14399                        vec![]
14400                    },
14401                )
14402                .unwrap();
14403        }
14404
14405        async fn probe(&mut self) -> crate::router::OutboundFrame {
14406            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14407                Ok(frame) => Some(frame),
14408                Err(mpsc::error::TryRecvError::Empty) => None,
14409                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14410            })
14411            .await;
14412            let body: Value = serde_json::from_slice(&frame.body).unwrap();
14413            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14414            frame
14415        }
14416
14417        fn no_probe(&mut self) {
14418            assert!(matches!(
14419                self.rx.try_recv(),
14420                Err(mpsc::error::TryRecvError::Empty)
14421            ));
14422        }
14423
14424        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14425            self.forwarding
14426                .complete_module_control_rpc(
14427                    ConnectionId::new(connection),
14428                    frame.header.corr,
14429                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14430                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14431                        status: HealthStatus::Ok,
14432                        detail: None,
14433                        metrics: None,
14434                    }),
14435                )
14436                .unwrap();
14437        }
14438    }
14439
14440    #[tokio::test(start_paused = true)]
14441    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14442        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14443        let initial = actor.module.status().unwrap();
14444        tokio::time::advance(Duration::from_secs(119)).await;
14445        tokio::task::yield_now().await;
14446        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14447        assert!(actor.module.terminal_history().entries.is_empty());
14448        actor.expire_registration_wait(Duration::from_secs(1)).await;
14449        actor
14450            .wait_for_select(|at| at.generation > initial.spawn_generation)
14451            .await;
14452        let status = actor.module.status().unwrap();
14453        assert_ne!(status.pid, initial.pid);
14454        assert!(!cortexkit_test_support::process_alive(
14455            initial.pid.unwrap() as i32
14456        ));
14457        assert_eq!(status.restart_count, 1);
14458        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14459        let entries = actor.module.terminal_history().entries;
14460        assert_eq!(entries.len(), 1);
14461        assert_eq!(
14462            entries[0].disposition_detail.as_deref(),
14463            Some("registration_timeout")
14464        );
14465        actor.module.stop().await.unwrap();
14466    }
14467
14468    #[tokio::test(start_paused = true)]
14469    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14470        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14471        let pid = actor.module.status().unwrap().pid;
14472        tokio::time::advance(Duration::from_secs(119)).await;
14473        actor.hello(1, false);
14474        let registered = actor
14475            .wait_for_select(|at| at.registered_connection.is_some())
14476            .await;
14477        assert_eq!(
14478            registered.wake_after, None,
14479            "HELLO must cancel the deadline"
14480        );
14481        tokio::time::advance(Duration::from_secs(300)).await;
14482        tokio::task::yield_now().await;
14483        assert_eq!(actor.module.status().unwrap().pid, pid);
14484        assert!(actor.module.terminal_history().entries.is_empty());
14485        // A disconnect after the first HELLO must not re-arm a startup timer.
14486        actor
14487            .registry
14488            .deregister_connection(ConnectionId::new(1))
14489            .unwrap();
14490        actor
14491            .wait_for_select(|at| at.registered_connection.is_none())
14492            .await;
14493        tokio::time::advance(Duration::from_secs(300)).await;
14494        tokio::task::yield_now().await;
14495        assert_eq!(actor.module.status().unwrap().pid, pid);
14496        assert!(actor.module.terminal_history().entries.is_empty());
14497        actor.module.stop().await.unwrap();
14498
14499        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14500        let pid = transient.module.status().unwrap().pid;
14501        tokio::time::advance(Duration::from_secs(119)).await;
14502        let turns = transient.turns();
14503        // Both writes happen before the supervisor can refresh the registry.
14504        // An accepted HELLO must remain evidence after its connection is gone.
14505        transient.hello(1, false);
14506        transient
14507            .registry
14508            .deregister_connection(ConnectionId::new(1))
14509            .unwrap();
14510        let parked = transient.wait_for_select(|at| at.turn > turns).await;
14511        assert_eq!(
14512            parked.wake_after, None,
14513            "transient HELLO must cancel the deadline"
14514        );
14515        tokio::time::advance(Duration::from_secs(300)).await;
14516        tokio::task::yield_now().await;
14517        assert_eq!(transient.module.status().unwrap().pid, pid);
14518        assert!(transient.module.terminal_history().entries.is_empty());
14519        transient.module.stop().await.unwrap();
14520    }
14521
14522    #[tokio::test(start_paused = true)]
14523    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14524        let actor = Actor::start_with_policy(
14525            ModuleProtocol::Subc,
14526            Duration::from_secs(30),
14527            Duration::from_secs(10),
14528            RestartPolicy::new(1, Duration::ZERO),
14529        )
14530        .await;
14531        let generation = actor.module.status().unwrap().spawn_generation;
14532        actor
14533            .expire_registration_wait(Duration::from_secs(10))
14534            .await;
14535        actor.wait_for_select(|at| at.generation > generation).await;
14536        actor
14537            .expire_registration_wait(Duration::from_secs(10))
14538            .await;
14539        let retry = wait_for("failed retry schedule", || {
14540            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14541            (state.state == ModuleState::Failed)
14542                .then_some(state.failed_retry.pending)
14543                .flatten()
14544        })
14545        .await;
14546        assert_eq!(retry.delay, Duration::from_secs(600));
14547        assert_eq!(retry.attempt, 1);
14548        let entries = actor.module.terminal_history().entries;
14549        assert_eq!(entries.len(), 2);
14550        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14551        assert!(entries[1]
14552            .disposition_detail
14553            .as_deref()
14554            .unwrap()
14555            .contains("registration_timeout"));
14556        tokio::time::advance(Duration::from_secs(599)).await;
14557        tokio::task::yield_now().await;
14558        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14559        tokio::time::advance(Duration::from_secs(1)).await;
14560        actor
14561            .wait_for_select(|at| at.generation > generation + 1)
14562            .await;
14563        // The slow retry is another ordinary spawn, with its own HELLO budget.
14564        actor
14565            .expire_registration_wait(Duration::from_secs(10))
14566            .await;
14567        wait_for("slow retry registration timeout", || {
14568            (actor.module.terminal_history().entries.len() == 3).then_some(())
14569        })
14570        .await;
14571        actor.module.stop().await.unwrap();
14572    }
14573
14574    #[tokio::test(start_paused = true)]
14575    async fn registration_timeout_honours_ten_second_override() {
14576        let actor = Actor::start_with_policy(
14577            ModuleProtocol::Subc,
14578            Duration::from_secs(30),
14579            Duration::from_secs(10),
14580            RestartPolicy::new(3, Duration::ZERO),
14581        )
14582        .await;
14583        let generation = actor.module.status().unwrap().spawn_generation;
14584        tokio::time::advance(Duration::from_secs(9)).await;
14585        tokio::task::yield_now().await;
14586        assert!(actor.module.terminal_history().entries.is_empty());
14587        actor.expire_registration_wait(Duration::from_secs(1)).await;
14588        actor.wait_for_select(|at| at.generation > generation).await;
14589        assert_eq!(actor.module.terminal_history().entries.len(), 1);
14590        actor.module.stop().await.unwrap();
14591    }
14592
14593    #[tokio::test]
14594    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14595        use std::os::unix::process::{CommandExt, ExitStatusExt};
14596
14597        struct Helper(std::process::Child);
14598        impl Drop for Helper {
14599            fn drop(&mut self) {
14600                // This handle still owns the helper until wait returns. Never
14601                // clean up its old process group after the leader is reaped.
14602                if matches!(self.0.try_wait(), Ok(None)) {
14603                    let _ = self.0.kill();
14604                    let _ = self.0.wait();
14605                }
14606            }
14607        }
14608
14609        let home = ScratchDir::new("registration-timeout-helper");
14610        let supervisor = Supervisor::new_for_test(
14611            Arc::new(Registry::default()),
14612            RestartPolicy::new(0, Duration::ZERO),
14613        );
14614        let spec = ModuleSpec {
14615            module_id: "registration-timeout-group".into(),
14616            program: "/bin/sleep".into(),
14617            args: vec!["120".into()],
14618            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14619                .into_iter()
14620                .map(|key| (key.into(), home.path().display().to_string()))
14621                .collect(),
14622            reserved: false,
14623            reserved_prefixes: vec![],
14624            protocol: ModuleProtocol::Subc,
14625            overlap: Default::default(),
14626        };
14627        let module = supervisor
14628            .supervise_configured_requested(
14629                spec,
14630                true,
14631                HealthConfig::default(),
14632                None,
14633                RestartPolicy::new(0, Duration::ZERO),
14634                Duration::from_secs(10),
14635                None,
14636            )
14637            .unwrap();
14638        let leader = timeout(Duration::from_secs(5), async {
14639            loop {
14640                if let Some(pid) = module.status().unwrap().pid {
14641                    break pid;
14642                }
14643                sleep(Duration::from_millis(5)).await;
14644            }
14645        })
14646        .await
14647        .expect("supervised group leader must be reported before the deadline");
14648        assert!(leader > 1);
14649        let ready = home.join("helper.ready");
14650        // Own the helper separately so even a broken group kill can be cleaned
14651        // up by its own unreaped child handle, without signalling a stale pgid.
14652        let mut helper = Helper(
14653            std::process::Command::new("/bin/sh")
14654                .args([
14655                    "-c",
14656                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14657                    "helper",
14658                ])
14659                .arg(&ready)
14660                .process_group(i32::try_from(leader).unwrap())
14661                .stdout(std::process::Stdio::null())
14662                .stderr(std::process::Stdio::null())
14663                .spawn()
14664                .unwrap(),
14665        );
14666        timeout(Duration::from_secs(5), async {
14667            while !ready.exists() {
14668                assert!(helper.0.try_wait().unwrap().is_none());
14669                sleep(Duration::from_millis(5)).await;
14670            }
14671        })
14672        .await
14673        .expect("helper must install TERM ignore before the registration deadline");
14674        let helper_pid =
14675            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14676        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14677        sleep(Duration::from_millis(20)).await;
14678        assert!(
14679            helper.0.try_wait().unwrap().is_none(),
14680            "helper must ignore TERM"
14681        );
14682
14683        timeout(Duration::from_secs(20), async {
14684            while module.state().unwrap() != ModuleState::Failed {
14685                sleep(Duration::from_millis(5)).await;
14686            }
14687        })
14688        .await
14689        .expect("registration timeout must stop and reap the group leader");
14690        let helper_exit = timeout(Duration::from_secs(3), async {
14691            loop {
14692                if let Some(status) = helper.0.try_wait().unwrap() {
14693                    break status;
14694                }
14695                sleep(Duration::from_millis(5)).await;
14696            }
14697        })
14698        .await
14699        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14700        assert_eq!(helper_exit.signal(), Some(9));
14701        assert!(!cortexkit_test_support::process_alive(
14702            i32::try_from(leader).unwrap()
14703        ));
14704        assert_eq!(
14705            module.status().unwrap().last_exit.unwrap().kind,
14706            ExitKind::Crash
14707        );
14708        assert!(module.terminal_history().entries[0]
14709            .disposition_detail
14710            .as_deref()
14711            .unwrap()
14712            .contains("registration_timeout"));
14713        module.stop().await.unwrap();
14714    }
14715
14716    #[tokio::test(start_paused = true)]
14717    async fn registration_timeout_does_not_count_an_earlier_exit() {
14718        let actor = Actor::start_with_policy(
14719            ModuleProtocol::Subc,
14720            Duration::from_secs(30),
14721            DEFAULT_REGISTER_TIMEOUT,
14722            RestartPolicy::new(0, Duration::ZERO),
14723        )
14724        .await;
14725        let pid = actor.module.status().unwrap().pid.unwrap();
14726        rustix::process::kill_process(
14727            rustix::process::Pid::from_raw(pid as i32).unwrap(),
14728            rustix::process::Signal::TERM,
14729        )
14730        .unwrap();
14731        wait_for("early exit", || {
14732            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14733        })
14734        .await;
14735        tokio::time::advance(Duration::from_secs(121)).await;
14736        tokio::task::yield_now().await;
14737        let entries = actor.module.terminal_history().entries;
14738        assert_eq!(entries.len(), 1);
14739        assert!(!entries[0]
14740            .disposition_detail
14741            .as_deref()
14742            .unwrap()
14743            .contains("registration_timeout"));
14744        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14745        actor.module.stop().await.unwrap();
14746    }
14747
14748    #[tokio::test(start_paused = true)]
14749    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14750        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14751        let pid = actor.module.status().unwrap().pid;
14752        tokio::time::advance(Duration::from_secs(119)).await;
14753        actor.roster.close();
14754        let turns = actor.turns();
14755        tokio::time::advance(Duration::from_secs(300)).await;
14756        let parked = actor.wait_for_select(|at| at.turn > turns).await;
14757        assert_eq!(parked.wake_after, None);
14758        assert_eq!(actor.module.status().unwrap().pid, pid);
14759        assert!(actor.module.terminal_history().entries.is_empty());
14760        actor.module.stop().await.unwrap();
14761        let status = actor.module.status().unwrap();
14762        assert_eq!(status.restart_count, 0);
14763        assert_eq!(
14764            actor.module.terminal_history().entries[0].disposition,
14765            TerminalDisposition::DaemonShutdown
14766        );
14767    }
14768
14769    #[tokio::test(start_paused = true)]
14770    async fn no_health_children_park_for_sixty_seconds() {
14771        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14772        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14773        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14774        unadvertised.hello(1, false);
14775        // Include any one-off registration wake in the budget; receiving a
14776        // no-health HELLO must not turn a parked child into a polling child.
14777        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
14778        // a single 60 s jump would only observe one expired timer.
14779        for _ in 0..6000 {
14780            tokio::time::advance(Duration::from_millis(10)).await;
14781            tokio::task::yield_now().await;
14782        }
14783        for actor in [&none, &unregistered, &unadvertised] {
14784            assert!(
14785                actor.turns() <= 5,
14786                "no-health actor woke {} times",
14787                actor.turns()
14788            );
14789            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14790        }
14791    }
14792
14793    #[tokio::test(start_paused = true)]
14794    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14795        // With a zero cadence the first probe is due at once, so this measures
14796        // only how fast the registration notification starts health probing,
14797        // not the normal 30-33 s wait before a first probe (checked below).
14798        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14799        let tick = Instant::now();
14800        actor.hello(1, true);
14801        actor.probe().await;
14802        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14803    }
14804
14805    #[tokio::test(start_paused = true)]
14806    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14807        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14808        let hello_tick = Instant::now();
14809        actor.hello(1, true);
14810        let armed = actor
14811            .wait_for_select(|checkpoint| {
14812                checkpoint.registered_connection == Some(ConnectionId::new(1))
14813                    && checkpoint.next_probe_at.is_some()
14814            })
14815            .await;
14816        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14817        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14818        let first_deadline = hello_tick + delay;
14819        assert_eq!(armed.next_probe_at, Some(first_deadline));
14820        tokio::time::advance(Duration::from_secs(29)).await;
14821        let turns = actor.turns();
14822        actor
14823            .registry
14824            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14825            .unwrap();
14826        let updated = actor
14827            .wait_for_select(|checkpoint| checkpoint.turn > turns)
14828            .await;
14829        assert_eq!(
14830            updated.next_probe_at,
14831            Some(first_deadline),
14832            "catalog update must not reset cadence"
14833        );
14834        actor.no_probe();
14835        tokio::time::advance(first_deadline - Instant::now()).await;
14836        let frame = actor.probe().await;
14837        actor.answer(1, frame);
14838        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14839        let next_deadline = Instant::now() + next;
14840        let rearmed = actor
14841            .wait_for_select(|checkpoint| {
14842                checkpoint
14843                    .next_probe_at
14844                    .is_some_and(|deadline| deadline > first_deadline)
14845            })
14846            .await;
14847        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14848        tokio::time::advance(next - Duration::from_nanos(1)).await;
14849        actor.no_probe();
14850        tokio::time::advance(Duration::from_nanos(1)).await;
14851        actor.probe().await;
14852    }
14853
14854    #[tokio::test(start_paused = true)]
14855    async fn disconnect_disarms_and_reconnect_rearms_health() {
14856        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14857        actor.hello(1, true);
14858        actor
14859            .wait_for_select(|checkpoint| {
14860                checkpoint.registered_connection == Some(ConnectionId::new(1))
14861                    && checkpoint.next_probe_at.is_some()
14862            })
14863            .await;
14864        let turns = actor.turns();
14865        let tick = Instant::now();
14866        actor
14867            .registry
14868            .deregister_connection(ConnectionId::new(1))
14869            .unwrap();
14870        let disarmed = actor
14871            .wait_for_select(|checkpoint| {
14872                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14873            })
14874            .await;
14875        assert_eq!(disarmed.next_probe_at, None);
14876        assert_eq!(disarmed.wake_after, None);
14877        assert_eq!(
14878            Instant::now(),
14879            tick,
14880            "disconnect must disarm in the notification tick"
14881        );
14882        tokio::time::advance(Duration::from_secs(60)).await;
14883        actor.no_probe();
14884        let reconnect_tick = Instant::now();
14885        actor.hello(2, true);
14886        let rearmed = actor
14887            .wait_for_select(|checkpoint| {
14888                checkpoint.registered_connection == Some(ConnectionId::new(2))
14889                    && checkpoint.next_probe_at.is_some()
14890            })
14891            .await;
14892        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14893        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14894        tokio::time::advance(delay).await;
14895        actor.probe().await;
14896    }
14897
14898    #[tokio::test(start_paused = true)]
14899    async fn reregistration_without_health_disarms_the_old_schedule() {
14900        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14901        actor.hello(1, true);
14902        actor
14903            .wait_for_select(|checkpoint| {
14904                checkpoint.registered_connection == Some(ConnectionId::new(1))
14905                    && checkpoint.next_probe_at.is_some()
14906            })
14907            .await;
14908        actor
14909            .registry
14910            .deregister_connection(ConnectionId::new(1))
14911            .unwrap();
14912        actor.hello(2, false);
14913        let disarmed = actor
14914            .wait_for_select(|checkpoint| {
14915                checkpoint.registered_connection == Some(ConnectionId::new(2))
14916            })
14917            .await;
14918        assert_eq!(disarmed.next_probe_at, None);
14919        assert_eq!(disarmed.wake_after, None);
14920        tokio::time::advance(Duration::from_secs(60)).await;
14921        actor.no_probe();
14922    }
14923
14924    #[tokio::test(start_paused = true)]
14925    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14926        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14927        actor.hello(1, true);
14928        let armed = actor
14929            .wait_for_select(|checkpoint| {
14930                checkpoint.registered_connection == Some(ConnectionId::new(1))
14931                    && checkpoint.next_probe_at.is_some()
14932            })
14933            .await;
14934        // Simulate the old process's connection closing before the restart
14935        // tears the process down. The replacement process has started but has
14936        // not sent its HELLO, so it is not registered.
14937        actor
14938            .registry
14939            .deregister_connection(ConnectionId::new(1))
14940            .unwrap();
14941        actor.module.restart(Some(0)).await.unwrap();
14942        let parked = actor
14943            .wait_for_select(|checkpoint| {
14944                checkpoint.generation > armed.generation
14945                    && checkpoint.registered_connection.is_none()
14946                    && checkpoint.next_probe_at.is_none()
14947            })
14948            .await;
14949        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
14950        actor.rx.close();
14951        // Drain the stop notice the restart queued for the old process's connection,
14952        // so it isn't mistaken for traffic to the replacement.
14953        while actor.rx.try_recv().is_ok() {}
14954        let turns = actor.turns();
14955        tokio::time::advance(Duration::from_secs(60)).await;
14956        let after = actor.turns();
14957        eprintln!(
14958            "restart park turns: before={turns} after={after} delta={}",
14959            after - turns
14960        );
14961        assert_eq!(after, turns, "replacement must park before HELLO");
14962        let hello_tick = Instant::now();
14963        actor.hello(2, true);
14964        let rearmed = actor
14965            .wait_for_select(|checkpoint| {
14966                checkpoint.registered_connection == Some(ConnectionId::new(2))
14967                    && checkpoint.next_probe_at.is_some()
14968            })
14969            .await;
14970        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14971        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14972        tokio::time::advance(delay).await;
14973        actor.probe().await;
14974    }
14975
14976    #[tokio::test(start_paused = true)]
14977    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14978        use tokio::{
14979            io::{AsyncReadExt, AsyncWriteExt},
14980            net::TcpListener,
14981        };
14982        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14983        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14984        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14985        let (started_tx, mut started_rx) = mpsc::channel(8);
14986        let peer = tokio::spawn(async move {
14987            loop {
14988                let (mut socket, _) = listener.accept().await.unwrap();
14989                let mut request = [0; 1024];
14990                assert!(socket.read(&mut request).await.unwrap() > 0);
14991                started_tx.send(Instant::now()).await.unwrap();
14992                socket
14993                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14994                    .await
14995                    .unwrap();
14996            }
14997        });
14998        actor
14999            .module
15000            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15001            .await
15002            .unwrap();
15003        let first_tick = Instant::now();
15004        let first = actor
15005            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15006            .await;
15007        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15008        assert!(started_rx.try_recv().is_err());
15009
15010        // Restart just before the old deadline: reusing it would give the new
15011        // process only one second to start, rather than a full cadence.
15012        tokio::time::advance(
15013            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15014        )
15015        .await;
15016        actor.module.restart(Some(0)).await.unwrap();
15017        let spawn_tick = Instant::now();
15018        let replacement = actor
15019            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15020            .await;
15021        let deadline = replacement.next_probe_at.unwrap();
15022        assert!(
15023            deadline >= spawn_tick + actor.health.cadence,
15024            "a replacement must not inherit its predecessor's probe deadline"
15025        );
15026        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15027        assert!(
15028            started_rx.try_recv().is_err(),
15029            "no HTTP probe during startup grace"
15030        );
15031        tokio::time::advance(deadline - Instant::now()).await;
15032        let landed = wait_for("first replacement HTTP probe", || {
15033            started_rx.try_recv().ok()
15034        })
15035        .await;
15036        assert!(landed >= spawn_tick + actor.health.cadence);
15037        wait_for("healthy replacement", || {
15038            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15039                .then_some(())
15040        })
15041        .await;
15042        peer.abort();
15043    }
15044
15045    #[tokio::test(start_paused = true)]
15046    async fn rescan_adds_and_removes_http_health_without_polling() {
15047        use tokio::{
15048            io::{AsyncReadExt, AsyncWriteExt},
15049            net::TcpListener,
15050        };
15051        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15052        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15053        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15054        // One millisecond separates successive probes without a polling delay.
15055        actor.health.cadence = Duration::from_millis(1);
15056        let (started_tx, mut started_rx) = mpsc::channel(8);
15057        let peer = tokio::spawn(async move {
15058            loop {
15059                let (mut socket, _) = listener.accept().await.unwrap();
15060                let mut request = [0; 1024];
15061                assert!(socket.read(&mut request).await.unwrap() > 0);
15062                started_tx.send(()).await.unwrap();
15063                socket
15064                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15065                    .await
15066                    .unwrap();
15067            }
15068        });
15069        let tick = Instant::now();
15070        let turns = actor.turns();
15071        actor
15072            .module
15073            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15074            .await
15075            .unwrap();
15076        let armed = actor
15077            .wait_for_select(|checkpoint| {
15078                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15079            })
15080            .await;
15081        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15082        assert_eq!(Instant::now(), tick);
15083        tokio::time::advance(Duration::from_millis(1)).await;
15084        let mut started = false;
15085        wait_for("successful HTTP probe", || {
15086            started |= started_rx.try_recv().is_ok();
15087            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15088                .then_some(())
15089        })
15090        .await;
15091        assert!(started, "added HTTP check must start at its first deadline");
15092        assert_eq!(
15093            actor.module.status().unwrap().health.status,
15094            SupervisorHealthStatus::Ok
15095        );
15096        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15097        actor.health.http = None;
15098        let turns = actor.turns();
15099        actor
15100            .module
15101            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15102            .await
15103            .unwrap();
15104        let parked = actor
15105            .wait_for_select(|checkpoint| {
15106                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15107            })
15108            .await;
15109        assert_eq!(parked.wake_after, None);
15110        let turns = actor.turns();
15111        tokio::time::advance(Duration::from_secs(60)).await;
15112        assert_eq!(
15113            actor.turns(),
15114            turns,
15115            "removed HTTP check must park the actor"
15116        );
15117        assert!(
15118            started_rx.try_recv().is_err(),
15119            "removed HTTP check must stay disarmed"
15120        );
15121        assert_eq!(
15122            actor.module.status().unwrap().health,
15123            ModuleHealthStatus::default()
15124        );
15125        peer.abort();
15126    }
15127}
15128
15129#[cfg(test)]
15130mod drain_busy_reprobe_tests {
15131    //! The drain's busy-gauge wait, driven against an in-process module
15132    //! connection whose health answers the test scripts. Time is paused, so
15133    //! every elapsed duration below is exact virtual time.
15134    use super::*;
15135    use std::collections::BTreeMap;
15136    use std::sync::Mutex as StdMutex;
15137    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15138
15139    const MODULE_ID: &str = "drain-reprobe-module";
15140    const CONNECTION: u64 = 7;
15141    const GAUGE: &str = "runs_in_flight";
15142    /// The production default for both the health cadence and the drain
15143    /// ceiling, which is the pairing that used to cost a full ceiling.
15144    const CADENCE: Duration = Duration::from_secs(30);
15145    const CEILING: Duration = Duration::from_secs(30);
15146
15147    /// How the scripted module answers each `health.check`.
15148    #[derive(Clone, Copy)]
15149    enum Answer {
15150        /// Report the gauge busy until this instant, quiet from then on, after
15151        /// taking `latency` to answer.
15152        BusyUntil { until: Instant, latency: Duration },
15153        /// Answer at once, with metrics that leave the declared gauge out.
15154        OmitsGauge,
15155        /// Never answer.
15156        Silent,
15157    }
15158
15159    struct Drain {
15160        forwarding: Arc<ForwardingTable>,
15161        runtime: SupervisorRuntimeConfig,
15162        endpoint: crate::ModuleEndpointId,
15163        /// When each probe reached the module.
15164        probes: Arc<StdMutex<Vec<Instant>>>,
15165        responder: tokio::task::JoinHandle<()>,
15166    }
15167
15168    impl Drain {
15169        fn start(answer: Answer) -> Self {
15170            let registry = Arc::new(Registry::default());
15171            let forwarding = Arc::new(ForwardingTable::default());
15172            let health = HealthConfig {
15173                cadence: CADENCE,
15174                ..HealthConfig::default()
15175            };
15176            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15177                .with_forwarding(forwarding.clone())
15178                .with_health_config(health)
15179                .runtime_config();
15180            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15181            let endpoint = forwarding
15182                .register_module_connection(
15183                    ConnectionId::new(CONNECTION),
15184                    MODULE_ID.to_string(),
15185                    subc_protocol::PROTOCOL_VERSION,
15186                    Concurrency::ModuleManaged,
15187                    FrameSink::new(tx),
15188                )
15189                .unwrap();
15190            let probes = Arc::new(StdMutex::new(Vec::new()));
15191            let responder = tokio::spawn({
15192                let forwarding = forwarding.clone();
15193                let probes = probes.clone();
15194                async move {
15195                    while let Some(frame) = rx.recv().await {
15196                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15197                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15198                        probes.lock().unwrap().push(Instant::now());
15199                        let metrics = match answer {
15200                            Answer::BusyUntil { until, latency } => {
15201                                sleep(latency).await;
15202                                let busy = u64::from(Instant::now() < until);
15203                                serde_json::json!({ GAUGE: busy })
15204                            }
15205                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15206                            Answer::Silent => continue,
15207                        };
15208                        // A probe abandoned at its deadline has no RPC left to
15209                        // complete, so a late answer may be refused.
15210                        let _ = forwarding.complete_module_control_rpc(
15211                            ConnectionId::new(CONNECTION),
15212                            frame.header.corr,
15213                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15214                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15215                                status: HealthStatus::Ok,
15216                                detail: None,
15217                                metrics: Some(metrics),
15218                            }),
15219                        );
15220                    }
15221                }
15222            });
15223            Self {
15224                forwarding,
15225                runtime,
15226                endpoint,
15227                probes,
15228                responder,
15229            }
15230        }
15231
15232        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15233        /// how long it took.
15234        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15235            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15236            (drained, elapsed)
15237        }
15238
15239        /// [`Self::run`], also returning what the drain's probes read.
15240        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15241            let started = Instant::now();
15242            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15243            let drained = wait_for_forwarding_quiescence(
15244                &self.forwarding,
15245                MODULE_ID,
15246                &self.runtime,
15247                self.endpoint,
15248                started + ceiling,
15249                &[GAUGE.to_string()],
15250                DrainScope::Active,
15251                &mut trace,
15252            )
15253            .await
15254            .unwrap();
15255            (drained, started.elapsed(), trace)
15256        }
15257
15258        fn probes(&self) -> Vec<Instant> {
15259            self.probes.lock().unwrap().clone()
15260        }
15261    }
15262
15263    impl Drop for Drain {
15264        fn drop(&mut self) {
15265            self.responder.abort();
15266        }
15267    }
15268
15269    #[tokio::test(start_paused = true)]
15270    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15271        let drain = Drain::start(Answer::BusyUntil {
15272            until: Instant::now() + Duration::from_secs(1),
15273            latency: Duration::ZERO,
15274        });
15275        let (drained, elapsed) = drain.run(CEILING).await;
15276        assert!(
15277            drained,
15278            "gauges that read zero during the drain must let it finish drained, \
15279             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15280        );
15281        assert!(
15282            elapsed < Duration::from_secs(3),
15283            "drain took {elapsed:?} after the gauges went quiet at 1s"
15284        );
15285        let probes = drain.probes();
15286        assert!(
15287            probes.len() >= 2,
15288            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15289        );
15290    }
15291
15292    /// The same shape in real time on a multi-threaded runtime: the first
15293    /// probe is answered busy at once, within a millisecond of the drain
15294    /// starting, and later probes read idle. The drain must keep probing
15295    /// after that first instant answer and end soon after the gauge falls.
15296    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15297    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15298        let drain = Drain::start(Answer::BusyUntil {
15299            until: Instant::now() + Duration::from_millis(600),
15300            latency: Duration::ZERO,
15301        });
15302        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15303        assert!(drained, "the gauge fell at 600ms: {trace}");
15304        assert!(
15305            elapsed < Duration::from_secs(3),
15306            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15307        );
15308        let probes = drain.probes();
15309        assert!(
15310            probes.len() >= 2 && trace.probes as usize == probes.len(),
15311            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15312        );
15313    }
15314
15315    #[tokio::test(start_paused = true)]
15316    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15317        let started = Instant::now();
15318        let drain = Drain::start(Answer::BusyUntil {
15319            until: started + Duration::from_secs(3600),
15320            latency: Duration::ZERO,
15321        });
15322        let (drained, elapsed) = drain.run(CEILING).await;
15323        assert!(!drained, "a module busy throughout must not drain");
15324        assert!(
15325            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15326            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15327        );
15328        // Re-probing is paced by the interval, not a busy loop: at most one
15329        // probe per interval across the ceiling, plus the first at time zero.
15330        let probes = drain.probes();
15331        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15332        assert!(
15333            probes.len() as u128 <= max_probes,
15334            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15335            probes.len()
15336        );
15337        for pair in probes.windows(2) {
15338            assert!(
15339                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15340                "consecutive probes {:?} apart",
15341                pair[1] - pair[0]
15342            );
15343        }
15344    }
15345
15346    #[tokio::test(start_paused = true)]
15347    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15348        // Each answer takes longer than the re-probe interval; the next probe
15349        // must wait for the previous answer and then the interval.
15350        let latency = Duration::from_secs(1);
15351        let drain = Drain::start(Answer::BusyUntil {
15352            until: Instant::now() + Duration::from_secs(3600),
15353            latency,
15354        });
15355        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15356        assert!(!drained);
15357        let probes = drain.probes();
15358        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15359        for pair in probes.windows(2) {
15360            assert!(
15361                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15362                "a probe started {:?} after the previous one, before it could finish",
15363                pair[1] - pair[0]
15364            );
15365        }
15366    }
15367
15368    #[tokio::test(start_paused = true)]
15369    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15370        let drain = Drain::start(Answer::Silent);
15371        let ceiling = Duration::from_secs(2);
15372        let (drained, elapsed) = drain.run(ceiling).await;
15373        assert!(!drained, "an unanswered probe counts as busy");
15374        assert!(
15375            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15376            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15377        );
15378        assert_eq!(
15379            drain.probes().len(),
15380            1,
15381            "the unanswered probe holds the only slot until the ceiling"
15382        );
15383    }
15384
15385    /// Every event logged while it is installed, as `message` plus fields.
15386    #[derive(Clone, Default)]
15387    struct LogCapture {
15388        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15389    }
15390
15391    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15392        fn on_event(
15393            &self,
15394            event: &tracing::Event<'_>,
15395            _context: tracing_subscriber::layer::Context<'_, S>,
15396        ) {
15397            struct Fields(BTreeMap<String, String>);
15398            impl tracing::field::Visit for Fields {
15399                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15400                    self.0
15401                        .insert(field.name().to_string(), format!("{value:?}"));
15402                }
15403            }
15404            let mut fields = Fields(BTreeMap::new());
15405            event.record(&mut fields);
15406            self.events.lock().unwrap().push(fields.0);
15407        }
15408    }
15409
15410    impl LogCapture {
15411        fn install(&self) -> tracing::subscriber::DefaultGuard {
15412            use tracing_subscriber::layer::SubscriberExt;
15413            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15414        }
15415
15416        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15417            self.events
15418                .lock()
15419                .unwrap()
15420                .iter()
15421                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15422                .cloned()
15423                .collect()
15424        }
15425    }
15426
15427    /// A module that declares a busy gauge but leaves it out of its health
15428    /// answers holds a drain to its ceiling. The timeout line must say so by
15429    /// name, and the drain must warn about the missing gauge once, not on
15430    /// every probe.
15431    #[tokio::test(start_paused = true)]
15432    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15433        let capture = LogCapture::default();
15434        let _guard = capture.install();
15435        let drain = Drain::start(Answer::OmitsGauge);
15436        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15437        assert!(!drained, "a gauge never reported cannot read quiet");
15438        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15439
15440        let missing = capture.with_message("lacks a declared busy gauge");
15441        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15442        assert_eq!(
15443            missing[0].get("missing_gauges").map(String::as_str),
15444            Some(GAUGE)
15445        );
15446
15447        warn_drain_timed_out(
15448            MODULE_ID,
15449            Duration::from_secs(2),
15450            RouteCloseReason::Restart,
15451            &Default::default(),
15452            &trace,
15453        );
15454        let timeouts = capture.with_message("route drain timed out");
15455        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15456        let gauges = &timeouts[0]["busy_gauges"];
15457        assert!(
15458            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15459            "the timeout line must name the gauge and why it read busy: {gauges}"
15460        );
15461        assert!(
15462            gauges.ends_with("; no value ever read"),
15463            "no value was ever read for the gauge: {gauges}"
15464        );
15465        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15466    }
15467
15468    /// The trace names a busy value and when it was read. For a probe that
15469    /// never got an answer, it names the probe's error kind and says that no
15470    /// value was ever read.
15471    #[tokio::test(start_paused = true)]
15472    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15473        let busy = Drain::start(Answer::BusyUntil {
15474            until: Instant::now() + Duration::from_secs(3600),
15475            latency: Duration::ZERO,
15476        });
15477        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15478        let rendered = trace.to_string();
15479        assert!(
15480            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15481                && rendered.contains("; last value non_zero(1) @"),
15482            "{rendered}"
15483        );
15484
15485        let silent = Drain::start(Answer::Silent);
15486        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15487        // The probe is capped at the 1 s ceiling, and its reading is stamped
15488        // when it gave up.
15489        assert_eq!(
15490            trace.to_string(),
15491            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15492        );
15493        assert_eq!(trace.probes, 1);
15494    }
15495
15496    /// A gauge value that is not a count reads busy and is shown as sent.
15497    #[test]
15498    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15499        let gauges = [GAUGE.to_string()];
15500        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15501        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15502        assert_eq!(
15503            busy_gauge_observation(&verdicts),
15504            BusyGaugeObservation::Busy
15505        );
15506        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15507        assert_eq!(
15508            busy_gauge_observation(&zero),
15509            BusyGaugeObservation::Quiescent
15510        );
15511        let none = gauge_verdicts(None, &gauges);
15512        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15513    }
15514}
15515
15516#[cfg(test)]
15517mod health_evidence_tests {
15518    use super::{HealthProbeError, HealthProbeEvidence};
15519    use std::collections::HashSet;
15520
15521    /// The evidential asymmetry, asserted rather than described.
15522    ///
15523    /// Exactly ONE observation is proof a module cannot serve, and the one that
15524    /// fires under CPU starvation is not it. Before the split, all fifteen
15525    /// construction sites collapsed into a single String, so a timeout carried the
15526    /// same weight as a dead lane -- which is how a healthy module was restarted
15527    /// three times in one day.
15528    #[test]
15529    fn only_a_dead_lane_is_proof_of_death() {
15530        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15531        // Three non-proof classes, each for a different reason: silence is
15532        // consistent with health, a bad answer proves the module ALIVE, and a
15533        // daemon-side fault never reached the module at all.
15534        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15535        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15536        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15537    }
15538
15539    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
15540    ///
15541    /// A shared label renders two different observations identically in the line an
15542    /// operator reads after an unexplained restart -- the exact confusion this
15543    /// change removes.
15544    #[test]
15545    fn every_evidence_class_has_a_distinct_label() {
15546        let labels = [
15547            HealthProbeError::lane_dead("").label(),
15548            HealthProbeError::no_answer("").label(),
15549            HealthProbeError::bad_answer("").label(),
15550            HealthProbeError::misconfigured("").label(),
15551        ];
15552        let unique: HashSet<_> = labels.iter().collect();
15553        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15554    }
15555
15556    /// The class is additional information, not a replacement.
15557    ///
15558    /// An operator needs both "this was silence" and the specific text saying how
15559    /// long we waited; a classification that swallowed the message would trade one
15560    /// missing distinction for another.
15561    #[test]
15562    fn classification_preserves_the_original_message() {
15563        let err = HealthProbeError::no_answer("module did not answer within 5s");
15564        assert_eq!(err.to_string(), "module did not answer within 5s");
15565        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15566    }
15567}
15568
15569#[cfg(test)]
15570mod health_tombstone_tests {
15571    use std::{path::PathBuf, sync::Arc, time::Duration};
15572
15573    use subc_protocol::{
15574        manifest::Concurrency,
15575        session::{HealthStatus, ModuleControlResponse},
15576    };
15577    use tokio::sync::mpsc;
15578
15579    use super::{
15580        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15581        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15582    };
15583    use crate::{
15584        control::ControlHandler,
15585        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15586        registry::{ConnectionId, Registry},
15587        router::FrameSink,
15588    };
15589
15590    struct ProbeHarness {
15591        spec: ModuleSpec,
15592        runtime: SupervisorRuntimeConfig,
15593        forwarding: Arc<ForwardingTable>,
15594        module_connection: ConnectionId,
15595        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15596        handler: ControlHandler,
15597        module: super::SupervisedModule,
15598    }
15599
15600    fn probe_harness() -> ProbeHarness {
15601        let registry = Arc::new(Registry::default());
15602        let forwarding = Arc::new(ForwardingTable::default());
15603        let supervisor_handle = super::SupervisorHandle::new();
15604        let health = HealthConfig {
15605            http: None,
15606            cadence: Duration::from_secs(30),
15607            deadline: Duration::from_secs(5),
15608            failure_threshold: 3,
15609            on_degraded: HealthAction::Report,
15610            on_failing: HealthAction::Report,
15611            critical: false,
15612        };
15613        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15614            .with_forwarding(Arc::clone(&forwarding))
15615            .with_handle(supervisor_handle.clone())
15616            .with_health_config(health);
15617        let spec = ModuleSpec {
15618            module_id: "late-health-module".to_string(),
15619            program: PathBuf::from("disabled-module"),
15620            args: Vec::new(),
15621            env: Vec::new(),
15622            reserved: false,
15623            reserved_prefixes: Vec::new(),
15624            protocol: ModuleProtocol::Subc,
15625            overlap: Default::default(),
15626        };
15627        let module = supervisor
15628            .supervise_configured(spec.clone(), false)
15629            .unwrap();
15630        let runtime = supervisor.runtime_config();
15631        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15632            .with_supervisor(supervisor_handle);
15633        let module_connection = ConnectionId::new(700);
15634        let (module_tx, module_rx) = mpsc::channel(8);
15635        forwarding
15636            .register_module_connection(
15637                module_connection,
15638                spec.module_id.clone(),
15639                subc_protocol::PROTOCOL_VERSION,
15640                Concurrency::ModuleManaged,
15641                FrameSink::new(module_tx),
15642            )
15643            .unwrap();
15644
15645        ProbeHarness {
15646            spec,
15647            runtime,
15648            forwarding,
15649            module_connection,
15650            module_rx,
15651            handler,
15652            module,
15653        }
15654    }
15655
15656    async fn finish_after(
15657        harness: &mut ProbeHarness,
15658        stall: Duration,
15659    ) -> ModuleControlRpcCompletion {
15660        assert!(stall > harness.runtime.health.deadline);
15661        let deadline = harness.runtime.health.deadline;
15662        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15663        let answer = async {
15664            let frame = harness.module_rx.recv().await.expect("health.check frame");
15665            tokio::time::advance(deadline).await;
15666            tokio::task::yield_now().await;
15667            tokio::time::advance(stall - deadline).await;
15668            harness
15669                .forwarding
15670                .complete_module_control_rpc(
15671                    harness.module_connection,
15672                    frame.header.corr,
15673                    Some("health.check"),
15674                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15675                        status: HealthStatus::Ok,
15676                        detail: None,
15677                        metrics: None,
15678                    }),
15679                )
15680                .unwrap()
15681        };
15682        let (probe_result, completion) = tokio::join!(probe, answer);
15683        let err = probe_result.expect_err("probe must miss its deadline");
15684        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15685        completion
15686    }
15687
15688    async fn time_out_without_answer(harness: &mut ProbeHarness) {
15689        let deadline = harness.runtime.health.deadline;
15690        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15691        let exhaust_deadline = async {
15692            let _frame = harness.module_rx.recv().await.expect("health.check frame");
15693            tokio::time::advance(deadline).await;
15694            tokio::task::yield_now().await;
15695        };
15696        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15697        let err = probe_result.expect_err("probe must miss its deadline");
15698        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15699    }
15700
15701    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15702        let registry = Arc::clone(&harness.module.inner.registry);
15703        let snapshot = Arc::clone(&harness.module.inner.snapshot);
15704        let process_liveness = super::SupervisorProcessLiveness::default();
15705        let mut child = None;
15706        let cycle = super::run_health_probe_cycle(
15707            &harness.spec,
15708            &harness.runtime,
15709            &registry,
15710            &process_liveness,
15711            &snapshot,
15712            &mut child,
15713        );
15714        let peer = async {
15715            let frame = harness.module_rx.recv().await.expect("health.check frame");
15716            if answer {
15717                harness
15718                    .forwarding
15719                    .complete_module_control_rpc(
15720                        harness.module_connection,
15721                        frame.header.corr,
15722                        Some("health.check"),
15723                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15724                            status: HealthStatus::Ok,
15725                            detail: None,
15726                            metrics: Some(serde_json::json!({"ready": true})),
15727                        }),
15728                    )
15729                    .unwrap();
15730            } else {
15731                tokio::time::advance(harness.runtime.health.deadline).await;
15732                tokio::task::yield_now().await;
15733            }
15734        };
15735        tokio::join!(cycle, peer);
15736    }
15737
15738    #[tokio::test(start_paused = true)]
15739    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15740        let mut harness = probe_harness();
15741        // Drive the probe cycle directly with an in-memory wire peer. Stop the
15742        // disabled module's monitor so only this test owns lifecycle transitions;
15743        // no OS process is launched, and a restart is observed at scheduling.
15744        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15745        monitor.abort();
15746        let _ = monitor.await;
15747        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15748            state.enabled = true;
15749            state.state = super::ModuleState::Running;
15750            state.process_alive = true;
15751        })
15752        .unwrap();
15753
15754        run_probe_cycle(&mut harness, true).await;
15755        assert_eq!(
15756            harness.module.status().unwrap().health.status,
15757            super::SupervisorHealthStatus::Ok
15758        );
15759
15760        for failures in 1..harness.runtime.health.failure_threshold {
15761            run_probe_cycle(&mut harness, false).await;
15762            let status = harness.module.status().unwrap();
15763            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15764            assert_eq!(status.health.consecutive_failures, failures);
15765            assert!(status.health.last_probe_ms.is_some());
15766            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15767            assert!(status.health.metrics.is_none());
15768            assert_eq!(status.state, super::ModuleState::Running);
15769            assert!(status.process_alive);
15770            assert_eq!(status.restart_count, 0);
15771            assert_eq!(status.lifetime_restarts, 0);
15772            assert!(status.health.last_action.is_none());
15773            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15774        }
15775
15776        run_probe_cycle(&mut harness, true).await;
15777        let recovered = harness.module.status().unwrap();
15778        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15779        assert_eq!(recovered.health.consecutive_failures, 0);
15780        assert!(recovered.health.detail.is_none());
15781        assert_eq!(
15782            recovered.health.metrics,
15783            Some(serde_json::json!({"ready": true}))
15784        );
15785        assert_eq!(recovered.lifetime_restarts, 0);
15786
15787        for failures in 1..=harness.runtime.health.failure_threshold {
15788            run_probe_cycle(&mut harness, false).await;
15789            let status = harness.module.status().unwrap();
15790            assert_eq!(status.health.consecutive_failures, failures);
15791            if failures < harness.runtime.health.failure_threshold {
15792                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15793                assert_eq!(status.state, super::ModuleState::Running);
15794                assert_eq!(status.lifetime_restarts, 0);
15795                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15796            } else {
15797                assert_eq!(
15798                    status.health.status,
15799                    super::SupervisorHealthStatus::Unresponsive
15800                );
15801                assert_eq!(status.state, super::ModuleState::Restarting);
15802                assert_eq!(status.restart_count, 1);
15803                assert_eq!(status.lifetime_restarts, 1);
15804                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15805                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15806            }
15807        }
15808    }
15809
15810    #[tokio::test(start_paused = true)]
15811    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15812        let mut harness = probe_harness();
15813
15814        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15815        let first_latency = match &first {
15816            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15817            other => panic!("late answer was not retained: {other:?}"),
15818        };
15819        assert!(harness.handler.observe_module_control_completion(first));
15820
15821        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15822        let second_latency = match &second {
15823            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15824            other => panic!("late answer was not retained: {other:?}"),
15825        };
15826        assert!(harness.handler.observe_module_control_completion(second));
15827
15828        assert_eq!(first_latency, Duration::from_secs(8));
15829        assert_eq!(
15830            second_latency - first_latency,
15831            Duration::from_secs(3),
15832            "latency must grow linearly with the additional stall"
15833        );
15834        let health = harness.module.status().unwrap().health;
15835        assert_eq!(health.late_answer_count, 2);
15836        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15837    }
15838
15839    /// A module that answers every probe late must never march to the kill
15840    /// threshold: the late answer proves it is alive, so it must clear the miss
15841    /// streak the timeout recorded. Without the reset, a CPU-starved module
15842    /// that serves every probe seconds past the deadline accumulates
15843    /// `consecutive_failures` to the threshold and is killed — the exact
15844    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15845    /// "proves the module is alive" five times while counting five misses.
15846    #[tokio::test(start_paused = true)]
15847    async fn late_answer_clears_the_consecutive_failure_streak() {
15848        let mut harness = probe_harness();
15849
15850        // Timeout recorded first: the probe path saw no answer in time.
15851        time_out_without_answer(&mut harness).await;
15852        harness
15853            .module
15854            .record_health_probe_failure_for_test("[no-answer] test miss")
15855            .unwrap();
15856        assert_eq!(
15857            harness.module.status().unwrap().health.consecutive_failures,
15858            1,
15859            "precondition: the miss must be on the streak before the late answer"
15860        );
15861
15862        // The stalled reply then lands: proof of life.
15863        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15864        assert!(matches!(
15865            late,
15866            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15867        ));
15868        assert!(harness.handler.observe_module_control_completion(late));
15869
15870        let health = harness.module.status().unwrap().health;
15871        assert_eq!(
15872            health.consecutive_failures, 0,
15873            "a late answer is an answer: the streak must reset"
15874        );
15875        assert_eq!(health.late_answer_count, 1);
15876    }
15877
15878    #[tokio::test(start_paused = true)]
15879    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15880        let mut harness = probe_harness();
15881
15882        for _ in 0..20 {
15883            time_out_without_answer(&mut harness).await;
15884            assert_eq!(
15885                harness.forwarding.health_probe_tombstone_count().unwrap(),
15886                1
15887            );
15888        }
15889    }
15890}
15891
15892#[cfg(test)]
15893mod child_env_tests {
15894    #[cfg(windows)]
15895    use super::{
15896        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15897    };
15898    #[cfg(windows)]
15899    use std::sync::{Arc, OnceLock};
15900    #[cfg(windows)]
15901    #[test]
15902    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15903        let spec = spec(Vec::new());
15904        let mut command = Command::new("unused");
15905        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15906            .unwrap()
15907            .unwrap();
15908        let name = command
15909            .as_std()
15910            .get_envs()
15911            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15912            .and_then(|(_, value)| value)
15913            .unwrap();
15914        assert_eq!(name, OsStr::new(handoff.name()));
15915        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15916        assert!(command.as_std().get_envs().any(|(key, value)| key
15917            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15918            && value == Some(OsStr::new("env"))));
15919        assert!(command
15920            .as_std()
15921            .get_envs()
15922            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15923    }
15924
15925    #[cfg(windows)]
15926    #[test]
15927    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15928        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15929        snapshot.process_alive = true;
15930        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15931        assert_eq!(
15932            observed_launch_nonce_source(&snapshot).as_deref(),
15933            Some("env")
15934        );
15935        let consumed = Arc::new(OnceLock::new());
15936        snapshot.pipe_consumed = Some(consumed.clone());
15937        assert_eq!(
15938            observed_launch_nonce_source(&snapshot).as_deref(),
15939            Some("env")
15940        );
15941        consumed.set(()).unwrap();
15942        assert_eq!(
15943            observed_launch_nonce_source(&snapshot).as_deref(),
15944            Some("pipe")
15945        );
15946        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15947        assert_eq!(
15948            observed_launch_nonce_source(&snapshot).as_deref(),
15949            Some("env")
15950        );
15951        clear_current_process_facts(&mut snapshot);
15952        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15953    }
15954    use super::{
15955        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15956        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15957        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15958    };
15959    use std::{ffi::OsStr, path::PathBuf};
15960    use tokio::process::Command;
15961
15962    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15963        ModuleSpec {
15964            module_id: "env-plan".to_string(),
15965            program: PathBuf::from("/nonexistent"),
15966            args: Vec::new(),
15967            env,
15968            reserved: false,
15969            reserved_prefixes: Vec::new(),
15970            protocol: ModuleProtocol::Subc,
15971            overlap: Default::default(),
15972        }
15973    }
15974
15975    #[test]
15976    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15977        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15978        let digest = LaunchSpecDigest::from_spec(&base);
15979        let mut reordered = base.clone();
15980        reordered.env.reverse();
15981        reordered.module_id = "different-id".into();
15982        reordered.reserved = true;
15983        reordered.overlap = Default::default();
15984        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15985        let mut delimiter_collision = base.clone();
15986        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15987        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15988        let mut changed = base.clone();
15989        changed.protocol = ModuleProtocol::None;
15990        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15991        changed = base.clone();
15992        changed.args.push("--flag".into());
15993        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15994        changed = base.clone();
15995        changed.program = PathBuf::from("/other");
15996        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15997        changed = base.clone();
15998        changed.env[1].1 = "other".into();
15999        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16000        assert!(!format!("{digest:?}").contains("B=C"));
16001        assert!(!format!("{digest:?}").contains("[u8"));
16002    }
16003
16004    #[cfg(unix)]
16005    #[test]
16006    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16007        use std::os::unix::ffi::OsStringExt;
16008        let mut left = spec(Vec::new());
16009        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16010        let mut right = left.clone();
16011        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16012        assert_eq!(
16013            left.program.to_string_lossy(),
16014            right.program.to_string_lossy()
16015        );
16016        assert_ne!(
16017            LaunchSpecDigest::from_spec(&left),
16018            LaunchSpecDigest::from_spec(&right)
16019        );
16020    }
16021
16022    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16023    /// one still gets its own.
16024    ///
16025    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16026    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16027    /// filter silently becoming an unconfigured module's log level is a real
16028    /// defect, just a much smaller one than clearing the environment.
16029    ///
16030    /// Asserted on the command plan rather than a spawned child because proving
16031    /// the ABSENCE of an inherited variable needs the parent's environment
16032    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16033    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16034    /// "absent because nobody set it" but "explicitly unset for the child".
16035    #[test]
16036    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16037        let mut command = Command::new("/nonexistent");
16038        apply_child_env(&mut command, &spec(Vec::new()));
16039        let removed = command
16040            .as_std()
16041            .get_envs()
16042            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16043        assert!(
16044            removed,
16045            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16046        );
16047
16048        let mut configured = Command::new("/nonexistent");
16049        apply_child_env(
16050            &mut configured,
16051            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16052        );
16053        let effective = configured
16054            .as_std()
16055            .get_envs()
16056            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16057            .last()
16058            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16059        assert_eq!(
16060            effective,
16061            Some(Some("debug".to_string())),
16062            "a module's configured CK_LOG must survive the ambient removal"
16063        );
16064    }
16065
16066    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16067    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16068    /// the same reason as the CK_LOG test above.
16069    ///
16070    /// The argument is the load-bearing half: a stock binary exits on an
16071    /// unknown flag before it listens, so with `--subc` appended the mode
16072    /// could not supervise the one process it exists for. Found by the first
16073    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16074    #[test]
16075    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16076        let connection_file = std::path::Path::new("/run/subc-connection.json");
16077        let handle = SupervisorHandle::new();
16078
16079        let mut none_spec = spec(Vec::new());
16080        none_spec.protocol = ModuleProtocol::None;
16081        let mut none = Command::new("/nonexistent");
16082        let none_handoff =
16083            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16084                .expect("protocol-none spawn args apply");
16085        assert!(
16086            none_handoff.is_none(),
16087            "protocol:none spawn must not receive a nonce descriptor"
16088        );
16089        #[cfg(windows)]
16090        assert!(!none.as_std().get_envs().any(|(key, value)| key
16091            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16092            && value.is_some()));
16093        assert!(
16094            !none.as_std().get_envs().any(|(key, value)| key
16095                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16096                && value.is_some()),
16097            "protocol:none spawn must not name a nonce descriptor"
16098        );
16099        let none_args: Vec<String> = none
16100            .as_std()
16101            .get_args()
16102            .map(|a| a.to_string_lossy().into_owned())
16103            .collect();
16104        assert!(
16105            !none_args.iter().any(|a| a == SUBC_ARG),
16106            "protocol:none argv must not carry --subc; got {none_args:?}"
16107        );
16108        let none_has_nonce = none
16109            .as_std()
16110            .get_envs()
16111            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16112        assert!(
16113            !none_has_nonce,
16114            "protocol:none spawn must not receive a launch nonce"
16115        );
16116        let none_has_module_id = none
16117            .as_std()
16118            .get_envs()
16119            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16120        assert!(
16121            none_has_module_id,
16122            "SUBC_MODULE_ID is inert and stays on every path"
16123        );
16124        assert!(
16125            handle.spawn_nonce(&none_spec.module_id).is_none(),
16126            "no nonce record for a process that will never present one"
16127        );
16128
16129        // Control: the subc-wire path is unchanged by the branch above.
16130        let wire_spec = spec(Vec::new());
16131        let mut wire = Command::new("/nonexistent");
16132        let wire_handoff =
16133            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16134                .expect("subc-wire spawn args apply");
16135        let wire_fd_env = wire
16136            .as_std()
16137            .get_envs()
16138            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16139            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16140        #[cfg(unix)]
16141        assert_eq!(
16142            wire_fd_env,
16143            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16144            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16145        );
16146        #[cfg(windows)]
16147        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16148        #[cfg(not(any(unix, windows)))]
16149        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16150        let wire_args: Vec<String> = wire
16151            .as_std()
16152            .get_args()
16153            .map(|a| a.to_string_lossy().into_owned())
16154            .collect();
16155        assert_eq!(
16156            wire_args,
16157            vec![
16158                SUBC_ARG.to_string(),
16159                connection_file.to_string_lossy().into_owned()
16160            ],
16161            "a subc-wire spawn still carries --subc <path>"
16162        );
16163        assert_eq!(
16164            wire.as_std()
16165                .get_envs()
16166                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16167            !cfg!(unix),
16168            "only Windows supplies the environment nonce"
16169        );
16170        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16171    }
16172
16173    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16174    /// spec tries to set it; only a swap candidate carries it.
16175    ///
16176    /// "Set it only on candidates" is not enough, because spawn applies the
16177    /// spec's env verbatim and the daemon's own environment is inherited: either
16178    /// could hand a plain restart the swap role, and a module reading it would
16179    /// warm on its long swap budget while callers wait. Asserted as an explicit
16180    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16181    /// test above gives.
16182    #[test]
16183    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16184        let role = |command: &Command| {
16185            command
16186                .as_std()
16187                .get_envs()
16188                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16189                .last()
16190                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16191        };
16192        let forged = spec(vec![(
16193            SUBC_SPAWN_ROLE_ENV.to_string(),
16194            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16195        )]);
16196
16197        let mut plain = Command::new("/nonexistent");
16198        apply_child_env(&mut plain, &forged);
16199        apply_spawn_role(&mut plain, SpawnRole::Plain);
16200        assert_eq!(
16201            role(&plain),
16202            Some(None),
16203            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16204        );
16205
16206        let mut candidate = Command::new("/nonexistent");
16207        apply_child_env(&mut candidate, &spec(Vec::new()));
16208        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16209        assert_eq!(
16210            role(&candidate),
16211            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16212        );
16213    }
16214
16215    /// Daemon-private capture retention keys never reach the child.
16216    ///
16217    /// cortexkit-log exposes retention as a Rust struct with no environment
16218    /// names, so these entries are supervisor metadata. Passing them through
16219    /// would invent a public child-process contract by accident.
16220    #[test]
16221    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16222        let mut command = Command::new("/nonexistent");
16223        apply_child_env(
16224            &mut command,
16225            &spec(vec![
16226                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16227                ("KEPT".to_string(), "yes".to_string()),
16228            ]),
16229        );
16230        let keys: Vec<String> = command
16231            .as_std()
16232            .get_envs()
16233            .filter(|(_, value)| value.is_some())
16234            .map(|(key, _)| key.to_string_lossy().into_owned())
16235            .collect();
16236        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16237        assert!(
16238            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16239            "daemon-private capture key leaked to the child: {keys:?}"
16240        );
16241    }
16242}
16243
16244#[cfg(test)]
16245mod jitter_tests {
16246    use super::jittered_health_delay;
16247    use std::{collections::HashSet, time::Duration};
16248
16249    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16250    /// that actually occur rather than invented ones.
16251    ///
16252    /// This is a SAMPLE, not a registry: the property under test is that distinct
16253    /// ids disperse, which holds for any set of distinct strings. Several entries
16254    /// are already historical (modules get renamed), and that costs nothing here --
16255    /// but it means a reader must not mistake this for the live module set, and a
16256    /// rename sweep will match it without there being anything to change.
16257    const FLEET: [&str; 14] = [
16258        "aft",
16259        "alfonso-core",
16260        "magic-context",
16261        "broca",
16262        "thalamus",
16263        "quota",
16264        "engram",
16265        "plexus",
16266        "cerebellum",
16267        "astrocyte",
16268        "synapse",
16269        "subc-mcp",
16270        "cortexkit-credentials",
16271        "subc-federation",
16272    ];
16273
16274    /// Probes must not converge after a fleet-wide restart.
16275    ///
16276    /// This is the property the jitter exists for: every module reconnects at
16277    /// once, and without dispersal all fourteen would then probe on the same
16278    /// tick forever. Nothing failed visibly when this went untested -- a
16279    /// convergent fleet still probes correctly, just in a burst, so the symptom
16280    /// is a periodic load spike that looks like whatever else is running.
16281    #[test]
16282    fn probe_delays_disperse_across_the_fleet() {
16283        let cadence = Duration::from_secs(30);
16284        let delays: HashSet<Duration> = FLEET
16285            .iter()
16286            .map(|id| jittered_health_delay(id, 0, cadence))
16287            .collect();
16288        assert_eq!(
16289            delays.len(),
16290            FLEET.len(),
16291            "every supervised module must land on its own probe offset"
16292        );
16293    }
16294
16295    /// The offset may only ever DELAY a probe, never bring it forward.
16296    ///
16297    /// A delay below the cadence would probe a module more often than
16298    /// configured, which is the opposite of what an operator asked for and
16299    /// would tighten the failure budget without anyone changing it.
16300    #[test]
16301    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16302        let cadence = Duration::from_secs(30);
16303        let span = cadence / 10;
16304        for id in FLEET {
16305            for probe_index in 0..8 {
16306                let delay = jittered_health_delay(id, probe_index, cadence);
16307                assert!(
16308                    delay >= cadence,
16309                    "{id}#{probe_index}: jitter must not shorten the cadence"
16310                );
16311                assert!(
16312                    delay < cadence + span,
16313                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16314                );
16315            }
16316        }
16317    }
16318
16319    /// A module keeps its offset across daemon restarts.
16320    ///
16321    /// The delay is derived rather than randomised precisely so a restart does
16322    /// not re-roll every module into a fresh chance of collision. A random
16323    /// source would satisfy the dispersal test above and quietly lose this.
16324    #[test]
16325    fn a_module_offset_is_stable_across_restarts() {
16326        let cadence = Duration::from_secs(30);
16327        for id in FLEET {
16328            assert_eq!(
16329                jittered_health_delay(id, 0, cadence),
16330                jittered_health_delay(id, 0, cadence),
16331                "{id}: the same module and probe index must produce the same offset"
16332            );
16333        }
16334    }
16335
16336    /// A zero cadence disables probing rather than producing a busy loop.
16337    #[test]
16338    fn zero_cadence_yields_zero_delay() {
16339        assert_eq!(
16340            jittered_health_delay("aft", 0, Duration::ZERO),
16341            Duration::ZERO
16342        );
16343    }
16344}
16345
16346#[cfg(all(test, target_os = "linux"))]
16347mod cgroup_placement_tests {
16348    use super::{
16349        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16350        SuperviseError, SupervisedChild,
16351    };
16352    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16353    use cortexkit_test_support::ScratchDir;
16354    use std::{
16355        fs, io,
16356        path::{Path, PathBuf},
16357        sync::{Arc, Mutex},
16358    };
16359    use tokio::process::Command;
16360
16361    #[tokio::test]
16362    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16363        use super::*;
16364        let dir = ScratchDir::new("unique-spawn-cgroups");
16365        let root = PathBuf::from(format!(
16366            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16367            std::process::id(),
16368            unix_ms_now()
16369        ));
16370        if let Err(error) = fs::create_dir(&root) {
16371            assert!(
16372                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16373                "required cgroup test cannot execute: {error}"
16374            );
16375            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16376            return;
16377        }
16378        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16379        let group_count = || {
16380            fs::read_dir(root.join("subc-modules"))
16381                .unwrap()
16382                .map(|entry| entry.unwrap().file_type().unwrap())
16383                .filter(|kind| kind.is_dir())
16384                .count()
16385        };
16386        let supervisor =
16387            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16388                .with_cgroup_placement(Some(placement.clone()));
16389        let runtime = supervisor.runtime_config();
16390        let mut spec = ModuleSpec {
16391            module_id: "unique-spawn".into(),
16392            program: PathBuf::from("/bin/sleep"),
16393            args: vec!["60".into()],
16394            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16395                .into_iter()
16396                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16397                .collect(),
16398            reserved: false,
16399            reserved_prefixes: vec![],
16400            protocol: ModuleProtocol::None,
16401            overlap: Default::default(),
16402        };
16403        let spawn = |spec: &ModuleSpec| {
16404            spawn_child(
16405                spec,
16406                None,
16407                None,
16408                &runtime.stderr_ring,
16409                None,
16410                &runtime.child_roster,
16411                Some(&placement),
16412            )
16413            .unwrap()
16414        };
16415        let mut live = spawn(&spec);
16416        for _ in 0..3 {
16417            // A new process can enter the old slot while retirement is pending.
16418            let next = spawn(&spec);
16419            assert_ne!(live.module_id, next.module_id);
16420            live.start_kill().unwrap();
16421            live.wait().await.unwrap();
16422            live = next;
16423            assert!(
16424                live.child.try_wait().unwrap().is_none(),
16425                "retiring the old slot must not kill the replacement"
16426            );
16427            assert_eq!(
16428                group_count(),
16429                1,
16430                "only the live spawn's cgroup should remain"
16431            );
16432        }
16433        supervisor.begin_daemon_shutdown();
16434        let reap = tokio::spawn(async move {
16435            live.wait().await.unwrap();
16436        });
16437        supervisor
16438            .end_children_for_daemon_shutdown(false, std::future::pending())
16439            .await;
16440        reap.await.unwrap();
16441        assert_eq!(group_count(), 0);
16442        // A normal exit uses the same tree-cleanup path as a killed spawn.
16443        spec.program = PathBuf::from("/bin/true");
16444        spec.args.clear();
16445        let fresh_roster = ChildRoster::default();
16446        let mut short = spawn_child(
16447            &spec,
16448            None,
16449            None,
16450            &runtime.stderr_ring,
16451            None,
16452            &fresh_roster,
16453            Some(&placement),
16454        )
16455        .unwrap();
16456        short.wait().await.unwrap();
16457        assert_eq!(group_count(), 0);
16458        spec.module_id = "_".repeat(255);
16459        let mut long_id = spawn_child(
16460            &spec,
16461            None,
16462            None,
16463            &runtime.stderr_ring,
16464            None,
16465            &fresh_roster,
16466            Some(&placement),
16467        )
16468        .unwrap();
16469        long_id.wait().await.unwrap();
16470        assert_eq!(
16471            group_count(),
16472            0,
16473            "valid long module IDs must not exceed cgroup NAME_MAX"
16474        );
16475        fs::remove_dir(root.join("subc-modules")).unwrap();
16476        fs::remove_dir(root).unwrap();
16477        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16478    }
16479
16480    #[test]
16481    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16482        let path = Path::new("/definitely-missing-subc-cgroup");
16483        let mut command = Command::new("true");
16484        let error = apply_cgroup_placement(
16485            &mut command,
16486            &ModuleSpec {
16487                module_id: "broken-cgroup".to_string(),
16488                program: PathBuf::from("true"),
16489                args: Vec::new(),
16490                env: Vec::new(),
16491                reserved: false,
16492                reserved_prefixes: Vec::new(),
16493                protocol: ModuleProtocol::Subc,
16494                overlap: Default::default(),
16495            },
16496            path,
16497        )
16498        .expect_err("a parent cgroup open failure must reject the supervised spawn");
16499        let reason = error.to_string();
16500
16501        assert!(
16502            matches!(error, SuperviseError::Cgroup { .. }),
16503            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16504        );
16505        assert!(
16506            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16507            "parent cgroup open failure must name cgroup.procs: {reason}"
16508        );
16509    }
16510
16511    #[tokio::test]
16512    async fn reaping_a_child_removes_its_empty_module_cgroup() {
16513        let root = ScratchDir::new("supervisor-reap-cgroup");
16514        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16515        let placement = subc_cgroup::prepare_at(&root)
16516            .expect("prepare scratch cgroup root")
16517            .expect("scratch root has a cgroup.procs marker");
16518        let module_id = "reaped-module";
16519        let module = placement
16520            .module_path(module_id)
16521            .expect("create scratch module cgroup");
16522        let child = Command::new("true")
16523            .env("XDG_DATA_HOME", root.path())
16524            .env("XDG_RUNTIME_DIR", root.path())
16525            .env("XDG_CONFIG_HOME", root.path())
16526            .spawn()
16527            .expect("spawn short-lived child");
16528        let pid = child.id().expect("spawned child has pid");
16529        let mut child = SupervisedChild {
16530            child,
16531            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16532            protocol: ModuleProtocol::Subc,
16533            module_id: module_id.to_string(),
16534            cgroup_placement: Some(placement),
16535            stdout_pump: None,
16536            stderr_pump: None,
16537            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16538            spawned_at: tokio::time::Instant::now(),
16539            registration_deadline: None,
16540            registration_watch: None,
16541            spawned_at_ms: 0,
16542            spawned_from: PathBuf::from("true"),
16543            launch_spec_digest: LaunchSpecDigest([0; 32]),
16544            spawned_file_identity: None,
16545            process_start_time: None,
16546            process_identity: None,
16547            pid,
16548            roster_guard: None,
16549            #[cfg(target_os = "macos")]
16550            privacy_exec: None,
16551            spawn_failure: None,
16552        };
16553
16554        child.wait().await.expect("reap short-lived child");
16555
16556        assert!(
16557            !module.exists(),
16558            "reaping the supervised child must remove its empty cgroup"
16559        );
16560    }
16561
16562    #[test]
16563    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16564        let root = ScratchDir::new("supervisor-non-empty-cgroup");
16565        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16566        let placement = subc_cgroup::prepare_at(&root)
16567            .expect("prepare scratch cgroup root")
16568            .expect("scratch root has a cgroup.procs marker");
16569        let module = placement
16570            .module_path("surviving-module")
16571            .expect("create scratch module cgroup");
16572        fs::write(module.join("surviving-process"), b"still present")
16573            .expect("make scratch cgroup non-empty");
16574        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16575
16576        remove_module_cgroup(&placement, "surviving-module");
16577
16578        let logs = crate::router::test_log::captured_logs(&logs);
16579        assert!(
16580            module.exists(),
16581            "failed removal must leave the cgroup intact"
16582        );
16583        assert!(
16584            logs.contains("could not remove module cgroup after process exit; continuing teardown")
16585                && logs.contains("surviving-module"),
16586            "best-effort removal must report the failure without returning it: {logs}"
16587        );
16588    }
16589
16590    #[test]
16591    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16592        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16593        let reason = SuperviseError::Spawn {
16594            program: PathBuf::from("/bin/true"),
16595            source: io::Error::from_raw_os_error(13),
16596            cgroup_path: Some(cgroup_path.clone()),
16597        }
16598        .to_string();
16599
16600        assert!(
16601            reason.contains(&cgroup_path.display().to_string()),
16602            "a pre_exec spawn failure must name the cgroup path: {reason}"
16603        );
16604    }
16605}
16606
16607#[cfg(test)]
16608mod spawn_subscriber_lag_tests {
16609    use super::*;
16610
16611    /// A subscriber whose connection stops draining is dropped once its frame
16612    /// channel fills. The client must learn that from a terminal Error frame
16613    /// after the frames already queued for it, not from a stream that simply
16614    /// goes quiet.
16615    #[tokio::test]
16616    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16617        let feed = SpawnEventFeed::default();
16618        feed.configure_incarnation("lag-incarnation".to_string());
16619        // A one-slot connection queue that nobody reads until the emits are
16620        // done: the forwarder parks on it and the subscriber channel fills.
16621        let (tx, mut rx) = mpsc::channel(1);
16622        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16623            .expect("subscribe");
16624        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16625        for index in 0..emitted {
16626            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16627            // Let the forwarder take what it can so the fill point is the
16628            // subscriber channel, not a scheduling accident.
16629            tokio::task::yield_now().await;
16630        }
16631        assert_eq!(
16632            feed.subscriber_count(),
16633            0,
16634            "the lagged subscriber must be removed"
16635        );
16636
16637        let mut data = Vec::new();
16638        let mut last = None;
16639        loop {
16640            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16641                .await
16642                .expect("the forwarder must finish once the subscriber is dropped");
16643            let Some(outbound) = next else { break };
16644            let frame = outbound.frame;
16645            if frame.header.ty == FrameType::StreamData {
16646                assert!(last.is_none(), "no data may follow the terminal frame");
16647                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16648                data.push(event.cursor.seq);
16649            } else {
16650                assert!(last.is_none(), "exactly one terminal frame");
16651                last = Some(frame);
16652            }
16653        }
16654        assert!(!data.is_empty(), "queued frames drain before the terminal");
16655        for pair in data.windows(2) {
16656            assert_eq!(
16657                pair[1],
16658                pair[0] + 1,
16659                "queued frames arrive dense and in order"
16660            );
16661        }
16662        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16663        assert_eq!(terminal.header.ty, FrameType::Error);
16664        assert_eq!(terminal.header.corr, 7);
16665        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16666        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16667        let detail = body.detail.expect("lagged error carries detail");
16668        assert_eq!(
16669            detail["first_undelivered_cursor"]["seq"],
16670            data.last().unwrap() + 1,
16671            "the named cursor is the first event the subscriber did not receive"
16672        );
16673        assert_eq!(
16674            detail["first_undelivered_cursor"]["daemon_incarnation"],
16675            "lag-incarnation"
16676        );
16677    }
16678}
16679
16680#[cfg(test)]
16681mod terminal_history_read_concurrency_tests {
16682    use super::*;
16683    use crate::terminal_journal::read_pause;
16684    use cortexkit_test_support::ScratchDir;
16685    use std::sync::mpsc as std_mpsc;
16686
16687    fn journaled_ring(
16688        journal: &Arc<crate::terminal_journal::TerminalJournal>,
16689    ) -> Arc<Mutex<TerminalRing>> {
16690        Arc::new(Mutex::new(
16691            TerminalRing::new(TerminalRingConfig::default(), 1)
16692                .with_journal(Some(Arc::clone(journal))),
16693        ))
16694    }
16695
16696    fn crash(at_ms: u64) -> ExitReport {
16697        ExitReport {
16698            kind: ExitKind::Crash,
16699            code: Some(1),
16700            signal: None,
16701            at_ms,
16702        }
16703    }
16704
16705    /// Record an exit on another thread and report whether it finished within
16706    /// `bound`. The recorder thread is left running if it did not.
16707    fn record_within(
16708        module_id: &'static str,
16709        ring: &Arc<Mutex<TerminalRing>>,
16710        at_ms: u64,
16711        bound: Duration,
16712    ) -> bool {
16713        let ring = Arc::clone(ring);
16714        let (done, done_rx) = std_mpsc::channel();
16715        std::thread::spawn(move || {
16716            record_terminal(
16717                module_id,
16718                &ring,
16719                &SpawnEventFeed::default(),
16720                &crash(at_ms),
16721                TerminalDisposition::Restarting,
16722            );
16723            let _ = done.send(());
16724        });
16725        done_rx.recv_timeout(bound).is_ok()
16726    }
16727
16728    /// A history read in progress must not hold the journal writer (which every
16729    /// module's exit recording needs) or the module's own ring. Exits recorded
16730    /// while the read is paused complete promptly; the paused read answers as of
16731    /// the moment it started, and the next read has each exit exactly once.
16732    #[test]
16733    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16734        let dir = ScratchDir::new("terminal-history-concurrent-read");
16735        let path = dir.join("terminals.jsonl");
16736        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16737            path.clone(),
16738            "daemon".into(),
16739        ));
16740        let reader_ring = journaled_ring(&journal);
16741        let other_ring = journaled_ring(&journal);
16742        assert!(record_within(
16743            "reader-module",
16744            &reader_ring,
16745            10,
16746            Duration::from_secs(5)
16747        ));
16748
16749        let (started, release) = read_pause::install(&path);
16750        let reading = {
16751            let ring = Arc::clone(&reader_ring);
16752            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16753        };
16754        started
16755            .recv_timeout(Duration::from_secs(5))
16756            .expect("the history read reached its pause");
16757
16758        let bound = Duration::from_secs(1);
16759        assert!(
16760            record_within("other-module", &other_ring, 20, bound),
16761            "another module's exit waited on a history read (journal writer held)"
16762        );
16763        assert!(
16764            record_within("reader-module", &reader_ring, 30, bound),
16765            "the read module's own exit waited on its history read (ring held)"
16766        );
16767
16768        drop(release);
16769        let paused = reading.join().unwrap();
16770        assert_eq!(
16771            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16772            vec![10],
16773            "an exit recorded after the read began lands in neither half of it"
16774        );
16775        assert_eq!(paused.journal_skipped_lines, 0);
16776        assert_eq!(paused.journal_read_errors, 0);
16777
16778        let after = durable_terminal_history_of(&reader_ring, "reader-module");
16779        assert_eq!(
16780            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16781            vec![10, 30],
16782            "the next read merges ring and journal with no duplicate"
16783        );
16784        assert_eq!(after.journal_skipped_lines, 0);
16785    }
16786}
16787
16788/// What a restart does with the exited process's output readers. These drive
16789/// the same settle functions the supervisor calls, with a reader the test
16790/// holds, so a reader that has not been scheduled by the bound is a controlled
16791/// input rather than something only a loaded machine produces.
16792#[cfg(test)]
16793mod stderr_settle_tests {
16794    use std::{
16795        future::Future,
16796        io,
16797        pin::Pin,
16798        sync::{Arc, Mutex},
16799        task::{Context, Poll},
16800        time::Duration,
16801    };
16802
16803    use tokio::{
16804        io::{AsyncRead, ReadBuf},
16805        sync::oneshot,
16806        time::Instant,
16807    };
16808
16809    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16810    use crate::stderr_tail::{
16811        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16812        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16813    };
16814
16815    const BOUND: Duration = Duration::from_millis(250);
16816
16817    /// Yields `before`, then stays pending until the gate is released, then
16818    /// yields `after` and reaches EOF. The bytes after the gate were written
16819    /// by a process that has already exited; only the reader is behind.
16820    struct HeldReader {
16821        before: Option<Vec<u8>>,
16822        gate: Option<oneshot::Receiver<()>>,
16823        after: io::Cursor<Vec<u8>>,
16824    }
16825
16826    impl AsyncRead for HeldReader {
16827        fn poll_read(
16828            mut self: Pin<&mut Self>,
16829            cx: &mut Context<'_>,
16830            buf: &mut ReadBuf<'_>,
16831        ) -> Poll<io::Result<()>> {
16832            if let Some(bytes) = self.before.take() {
16833                buf.put_slice(&bytes);
16834                return Poll::Ready(Ok(()));
16835            }
16836            if let Some(gate) = self.gate.as_mut() {
16837                match Pin::new(gate).poll(cx) {
16838                    Poll::Pending => return Poll::Pending,
16839                    Poll::Ready(_) => self.gate = None,
16840                }
16841            }
16842            Pin::new(&mut self.after).poll_read(cx, buf)
16843        }
16844    }
16845
16846    struct DiscardSink;
16847
16848    impl OutputSink for DiscardSink {
16849        fn write_line(&mut self, _line: &[u8]) {}
16850    }
16851
16852    fn line(text: &str) -> TailEntry {
16853        TailEntry::Line {
16854            text: text.to_string(),
16855            truncated: false,
16856            at_ms: None,
16857        }
16858    }
16859
16860    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16861        ring.lock().unwrap()
16862    }
16863
16864    /// Start a reader for a new process generation that delivers `before`
16865    /// immediately and `after` only once the returned sender fires (or is
16866    /// dropped).
16867    fn held_pump(
16868        ring: &Arc<Mutex<StderrRing>>,
16869        before: &str,
16870        after: &str,
16871    ) -> (StderrPump, oneshot::Sender<()>) {
16872        let generation = lock(ring).begin_process();
16873        let (release, gate) = oneshot::channel();
16874        let reader = HeldReader {
16875            before: Some(before.as_bytes().to_vec()),
16876            gate: Some(gate),
16877            after: io::Cursor::new(after.as_bytes().to_vec()),
16878        };
16879        let task = tokio::spawn(pump_stderr_to(
16880            reader,
16881            Arc::clone(ring),
16882            generation,
16883            DiscardSink,
16884        ));
16885        (StderrPump { task, generation }, release)
16886    }
16887
16888    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16889        for _ in 0..1000 {
16890            if done(&lock(ring)) {
16891                return;
16892            }
16893            tokio::time::sleep(Duration::from_millis(1)).await;
16894        }
16895        panic!(
16896            "ring never reached the expected state: {:?}",
16897            lock(ring).snapshot(None, None)
16898        );
16899    }
16900
16901    #[tokio::test(start_paused = true)]
16902    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16903        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16904        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16905
16906        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16907        let before_release = lock(&ring).snapshot(None, None);
16908        assert!(
16909            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16910            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16911        );
16912
16913        // The restart: the next process starts and writes before the old
16914        // reader catches up.
16915        let next = lock(&ring).begin_process();
16916        lock(&ring).push_line_from(next, "next process booting");
16917        release.send(()).unwrap();
16918        wait_until(&ring, |ring| {
16919            ring.snapshot(None, None).capture == CaptureState::Captured
16920        })
16921        .await;
16922
16923        assert_eq!(
16924            untimed(lock(&ring).snapshot(None, None).entries),
16925            vec![
16926                line("booting"),
16927                line("config error: missing storage"),
16928                TailEntry::ProcessStart,
16929                line("next process booting"),
16930            ],
16931            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16932        );
16933    }
16934
16935    #[tokio::test(start_paused = true)]
16936    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16937    ) {
16938        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16939        // `_held` is never fired: a descendant keeps the pipe open for the
16940        // whole test.
16941        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16942
16943        let started = Instant::now();
16944        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16945        assert_eq!(
16946            started.elapsed(),
16947            BOUND,
16948            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16949        );
16950
16951        let next = lock(&ring).begin_process();
16952        lock(&ring).push_line_from(next, "next process booting");
16953        tokio::time::sleep(Duration::from_secs(60)).await;
16954
16955        let snapshot = lock(&ring).snapshot(None, None);
16956        match &snapshot.capture {
16957            CaptureState::Incomplete { reason } => assert!(
16958                reason.contains("had not reached EOF") && reason.contains("250ms"),
16959                "the reason must say what is missing and after how long: {reason}"
16960            ),
16961            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16962        }
16963        assert_eq!(
16964            untimed(snapshot.entries),
16965            vec![
16966                line("parent exiting"),
16967                TailEntry::ProcessStart,
16968                line("next process booting"),
16969            ]
16970        );
16971    }
16972
16973    #[tokio::test(start_paused = true)]
16974    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16975        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16976        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16977        release.send(()).unwrap();
16978
16979        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16980
16981        let snapshot = lock(&ring).snapshot(None, None);
16982        assert_eq!(snapshot.capture, CaptureState::Captured);
16983        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16984    }
16985
16986    #[tokio::test(start_paused = true)]
16987    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16988        const BURST: usize = 150;
16989        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16990        let path = capture.join("two-pipe.stderr.log");
16991        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16992        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16993        let generation = lock(&ring).begin_process();
16994        let stdout = (0..BURST)
16995            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16996            .collect::<String>();
16997        let stderr = (0..BURST)
16998            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16999            .collect::<String>();
17000        // Model a child that has written both bursts and exited. Hold the stdout
17001        // reader after one line and a partial prefix, as if pipe readiness or
17002        // capture-file writes had delayed it. The final line also needs EOF to
17003        // flush: neither complete nor partial buffered output may be discarded.
17004        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17005        let (release, gate) = oneshot::channel();
17006        let reader = HeldReader {
17007            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17008            gate: Some(gate),
17009            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17010        };
17011        let (finished, completion) = oneshot::channel();
17012        let stdout_sink = sink.clone();
17013        let pump = tokio::spawn(async move {
17014            pump_stdout_to(reader, stdout_sink).await;
17015            finished.send(()).unwrap();
17016        });
17017        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17018
17019        let started = Instant::now();
17020        settle_stdout_pump("two-pipe", pump, BOUND).await;
17021        assert_eq!(
17022            started.elapsed(),
17023            BOUND,
17024            "a late reader must not hold up restart"
17025        );
17026        assert_eq!(
17027            std::fs::read_to_string(&path).unwrap().lines().count(),
17028            BURST + 1
17029        );
17030
17031        release
17032            .send(())
17033            .expect("stdout reader must survive the exit drain bound");
17034        tokio::time::timeout(Duration::from_secs(1), completion)
17035            .await
17036            .expect("released stdout reader must reach EOF")
17037            .expect("stdout capture must finish at EOF, not be cancelled");
17038        let contents = std::fs::read_to_string(&path).unwrap();
17039        let mut lines = contents
17040            .lines()
17041            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17042            .collect::<Vec<_>>();
17043        let mut expected = (0..BURST)
17044            .flat_map(|i| {
17045                [
17046                    format!("out-{i:04}-{}", "o".repeat(64)),
17047                    format!("err-{i:04}-{}", "e".repeat(64)),
17048                ]
17049            })
17050            .collect::<Vec<_>>();
17051        lines.sort_unstable();
17052        expected.sort_unstable();
17053        assert_eq!(
17054            lines, expected,
17055            "both pipes must retain every complete line exactly once"
17056        );
17057    }
17058}
17059
17060/// Containment of a module's process tree (issue #109).
17061///
17062/// The behaviour these defend against is a module helper surviving its module:
17063/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17064/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17065/// compounds it.
17066///
17067/// They run against the SUPERVISOR rather than the job-object crate because the
17068/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17069/// that the daemon's drain path reaches it.
17070///
17071/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17072/// lane there is a separate containment path with its own tests.
17073#[cfg(all(test, windows))]
17074mod job_containment_tests {
17075    use super::*;
17076    use cortexkit_test_support::ScratchDir;
17077    use std::{
17078        path::{Path, PathBuf},
17079        sync::{Arc, Mutex},
17080        time::{Duration, Instant},
17081    };
17082
17083    /// The stub, expected beside this test executable.
17084    ///
17085    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17086    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17087    /// failure then reads as a broken test rather than an unbuilt dependency.
17088    fn stub_path() -> PathBuf {
17089        let mut path = std::env::current_exe().expect("current_exe available in tests");
17090        path.pop();
17091        path.pop();
17092        path.push("fake-aft-stub.exe");
17093        assert!(
17094            path.exists(),
17095            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17096             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17097            path.display()
17098        );
17099        path
17100    }
17101
17102    /// Poll for the grandchild pid the stub records, and parse it.
17103    fn read_grandchild_pid(path: &Path) -> u32 {
17104        let deadline = Instant::now() + Duration::from_secs(10);
17105        loop {
17106            if let Ok(contents) = std::fs::read_to_string(path) {
17107                if let Ok(pid) = contents.trim().parse() {
17108                    return pid;
17109                }
17110            }
17111            assert!(
17112                Instant::now() < deadline,
17113                "the stub never recorded a grandchild pid at {}",
17114                path.display()
17115            );
17116            std::thread::sleep(Duration::from_millis(10));
17117        }
17118    }
17119
17120    /// Everything one fixture run needs, so the two tests below differ in exactly
17121    /// one place: whether the child is contained.
17122    struct Fixture {
17123        _dir: ScratchDir,
17124        module_id: String,
17125        grandchild: u32,
17126        child: Option<SupervisedChild>,
17127        registry: Arc<Registry>,
17128        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17129        terminal_ring: Arc<Mutex<TerminalRing>>,
17130        spawn_events: SpawnEventFeed,
17131    }
17132
17133    fn fixture(label: &str, module_id: &str) -> Fixture {
17134        let dir = ScratchDir::new(label);
17135        let pid_file = dir.join("grandchild.pid");
17136        let supervisor = Supervisor::new_for_test(
17137            Arc::new(Registry::default()),
17138            RestartPolicy::new(3, Duration::ZERO),
17139        );
17140        let runtime = supervisor.runtime_config();
17141        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17142        let spec = ModuleSpec {
17143            module_id: module_id.to_string(),
17144            program: stub_path(),
17145            // Zero args deliberately: a `--subc` argument would make the stub dial
17146            // a daemon that is not there, and the failure would land in the same
17147            // stderr ring this fixture exists to keep quiet.
17148            args: Vec::new(),
17149            env: vec![
17150                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17151                (
17152                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17153                    pid_file.display().to_string(),
17154                ),
17155            ],
17156            reserved: false,
17157            reserved_prefixes: Vec::new(),
17158            protocol: ModuleProtocol::Subc,
17159            overlap: Default::default(),
17160        };
17161        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17162            .expect("spawn the supervised fixture");
17163        let grandchild = read_grandchild_pid(&pid_file);
17164        Fixture {
17165            _dir: dir,
17166            module_id: module_id.to_string(),
17167            grandchild,
17168            child: Some(child),
17169            registry: Arc::new(Registry::default()),
17170            snapshot,
17171            terminal_ring: Arc::clone(&runtime.terminal_ring),
17172            spawn_events: SpawnEventFeed::default(),
17173        }
17174    }
17175
17176    impl Fixture {
17177        /// Drain through the supervisor's own teardown path.
17178        async fn drain(&mut self) {
17179            let child = self
17180                .child
17181                .take()
17182                .expect("the fixture child is still present");
17183            drain_child_to_state(
17184                &self.module_id,
17185                ModuleProtocol::Subc,
17186                // No forwarding table in this fixture, so nothing reaches the
17187                // child over a connection.
17188                StopNotice::NotSent,
17189                &self.registry,
17190                None,
17191                &self.snapshot,
17192                &self.terminal_ring,
17193                &self.spawn_events,
17194                child,
17195                Duration::from_millis(500),
17196                ModuleState::Stopped,
17197                Some(false),
17198                &FailureContext::operator(),
17199            )
17200            .await
17201            .expect("drain the supervised fixture");
17202        }
17203    }
17204
17205    /// Teardown reaps the grandchild, not merely the direct child.
17206    ///
17207    /// This is the assertion the change exists for. Before containment the
17208    /// grandchild survived: it is a separate process, and `start_kill` is
17209    /// `TerminateProcess` scoped to one pid.
17210    #[tokio::test]
17211    async fn teardown_reaps_the_grandchild() {
17212        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17213        let grandchild = fixture.grandchild;
17214
17215        assert!(
17216            subc_jobobject::process_exists(grandchild),
17217            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17218        );
17219
17220        fixture.drain().await;
17221
17222        assert!(
17223            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17224            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17225        );
17226    }
17227
17228    /// The mutation control: with containment withheld, the grandchild survives
17229    /// the same kill.
17230    ///
17231    /// This is the defect reproduction from #109 — a direct-child kill reaches
17232    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17233    /// supervisor because `spawn_and_mark_running` now always contains on
17234    /// Windows, which is the point: there is no longer a path that spawns
17235    /// uncontained, so the control has to construct one.
17236    ///
17237    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17238    /// grandchild ever dies here, that test is passing for a reason unrelated to
17239    /// the job object and the containment claim is unproven.
17240    #[test]
17241    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17242        let dir = ScratchDir::new("teardown-uncontained");
17243        let pid_file = dir.join("grandchild.pid");
17244        let mut child = std::process::Command::new(stub_path())
17245            .env("FAKE_AFT_NEVER_CONNECT", "1")
17246            .env(
17247                "FAKE_AFT_GRANDCHILD_PID_FILE",
17248                pid_file.display().to_string(),
17249            )
17250            .stdin(std::process::Stdio::null())
17251            .stdout(std::process::Stdio::null())
17252            .stderr(std::process::Stdio::null())
17253            .spawn()
17254            .expect("spawn the uncontained fixture");
17255        let grandchild = read_grandchild_pid(&pid_file);
17256
17257        // Exactly what the pre-fix teardown did: kill the direct child.
17258        child.kill().expect("kill the direct child");
17259        let _ = child.wait();
17260
17261        assert!(
17262            subc_jobobject::process_exists(grandchild),
17263            "grandchild {grandchild} died with the direct child, so this control no longer \
17264             distinguishes contained from uncontained teardown and the regression test is \
17265             passing vacuously"
17266        );
17267
17268        // The orphan this control demonstrates is the leak the fix prevents, so
17269        // the control must not leave one behind.
17270        kill_tree(grandchild);
17271    }
17272
17273    /// Crash durability: closing the containment handle reaps the tree with no
17274    /// teardown code running at all.
17275    ///
17276    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17277    /// call anything — and it is why containment is a kernel property of the
17278    /// handle rather than a step in the drain. Discovered by getting the
17279    /// mutation control wrong: clearing `job` to "disable" containment instead
17280    /// killed the tree, which is the guarantee, not a mistake.
17281    #[tokio::test]
17282    async fn dropping_containment_reaps_the_grandchild() {
17283        let mut fixture = fixture("drop-containment", "tree-drop");
17284        let grandchild = fixture.grandchild;
17285
17286        assert!(subc_jobobject::process_exists(grandchild));
17287
17288        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17289        fixture.child.as_mut().expect("child present").job = None;
17290
17291        assert!(
17292            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17293            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17294             crash would leave the tree behind"
17295        );
17296    }
17297
17298    /// Kill a pid and its tree, then confirm it is gone.
17299    fn kill_tree(pid: u32) {
17300        let _ = std::process::Command::new("taskkill.exe")
17301            .args(["/PID", &pid.to_string(), "/T", "/F"])
17302            .stdin(std::process::Stdio::null())
17303            .stdout(std::process::Stdio::null())
17304            .stderr(std::process::Stdio::null())
17305            .status();
17306        assert!(
17307            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17308            "could not clean up grandchild {pid}"
17309        );
17310    }
17311}
17312
17313#[cfg(test)]
17314mod privacy_trampoline_configuration_tests {
17315    #[tokio::test]
17316    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17317    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17318        #[cfg(target_os = "macos")]
17319        {
17320            let supervisor = super::Supervisor::new(
17321                std::sync::Arc::new(crate::Registry::default()),
17322                super::RestartPolicy::default(),
17323            );
17324            let error = supervisor.spawn(spec()).unwrap_err();
17325            assert!(
17326                error
17327                    .to_string()
17328                    .contains("no privacy trampoline configured"),
17329                "{error}"
17330            );
17331        }
17332    }
17333
17334    #[tokio::test]
17335    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17336    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17337        #[cfg(target_os = "macos")]
17338        {
17339            let supervisor = super::Supervisor::new(
17340                std::sync::Arc::new(crate::Registry::default()),
17341                super::RestartPolicy::default(),
17342            )
17343            .with_privacy_trampoline(std::env::current_exe().unwrap());
17344            let error = supervisor.spawn(spec()).unwrap_err();
17345            assert!(
17346                error
17347                    .to_string()
17348                    .contains("binary does not implement the privacy trampoline protocol"),
17349                "{error}"
17350            );
17351        }
17352    }
17353
17354    #[cfg(target_os = "macos")]
17355    fn spec() -> super::ModuleSpec {
17356        super::ModuleSpec {
17357            module_id: "privacy-configuration".into(),
17358            program: "/bin/sleep".into(),
17359            args: vec!["30".into()],
17360            env: vec![],
17361            reserved: false,
17362            reserved_prefixes: vec![],
17363            protocol: subc_control::ModuleProtocol::None,
17364            overlap: super::ModuleOverlap::Exclusive,
17365        }
17366    }
17367}
17368
17369#[cfg(test)]
17370mod privacy_exec_boundary_tests {
17371    #[cfg(target_os = "macos")]
17372    use super::*;
17373    #[cfg(target_os = "macos")]
17374    use std::{
17375        io::{Read, Write},
17376        net::{TcpListener, TcpStream},
17377    };
17378
17379    /// Unit-test-only pause at the actual early image read, not at a later
17380    /// status read. Production supervisors never inspect this environment key.
17381    #[cfg(target_os = "macos")]
17382    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17383        if let Some((_, path)) = spec
17384            .env
17385            .iter()
17386            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17387        {
17388            let mut barrier = TcpStream::connect(path).unwrap();
17389            barrier
17390                .set_read_timeout(Some(Duration::from_secs(30)))
17391                .unwrap();
17392            barrier.write_all(&pid.to_ne_bytes()).unwrap();
17393            let mut release = [0];
17394            barrier.read_exact(&mut release).unwrap();
17395            assert_eq!(&release, b"X");
17396        }
17397    }
17398
17399    #[cfg(target_os = "macos")]
17400    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17401        ModuleSpec {
17402            module_id: "privacy-boundary".into(),
17403            program: program.into(),
17404            args: args.iter().map(|arg| (*arg).into()).collect(),
17405            env: vec![],
17406            reserved: false,
17407            reserved_prefixes: vec![],
17408            protocol: ModuleProtocol::None,
17409            overlap: ModuleOverlap::Exclusive,
17410        }
17411    }
17412
17413    #[cfg(target_os = "macos")]
17414    async fn accept(listener: TcpListener) -> TcpStream {
17415        // Socket readiness, not elapsed time, establishes both pause points.
17416        let listener = tokio::net::TcpListener::from_std({
17417            listener.set_nonblocking(true).unwrap();
17418            listener
17419        })
17420        .unwrap();
17421        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17422            .await
17423            .unwrap()
17424            .unwrap();
17425        let stream = stream.into_std().unwrap();
17426        stream.set_nonblocking(false).unwrap();
17427        stream
17428            .set_read_timeout(Some(Duration::from_secs(30)))
17429            .unwrap();
17430        stream
17431    }
17432
17433    #[tokio::test]
17434    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17435    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17436        #[cfg(target_os = "macos")]
17437        {
17438            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17439            // Loopback sockets also work when the replay adapter's TMPDIR is
17440            // longer than Darwin's Unix-domain socket path limit.
17441            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17442            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17443            let record = root.join("live-children.json");
17444            let supervisor =
17445                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17446                    .with_live_children_record(&record);
17447            let runtime = supervisor.runtime_config();
17448            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17449            let mut spec = spec("/bin/sleep", &["30"]);
17450            spec.env = vec![
17451                (
17452                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17453                    exec_listener.local_addr().unwrap().to_string(),
17454                ),
17455                (
17456                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17457                    sample_listener.local_addr().unwrap().to_string(),
17458                ),
17459            ];
17460            let spawn = tokio::task::spawn_blocking(move || {
17461                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17462            });
17463            let mut sample = accept(sample_listener).await;
17464            let mut pid = [0; 4];
17465            sample.read_exact(&mut pid).unwrap();
17466            let pid = u32::from_ne_bytes(pid);
17467            let mut exec = accept(exec_listener).await;
17468            let mut ready = [0];
17469            exec.read_exact(&mut ready).unwrap();
17470            assert_eq!(&ready, b"R");
17471            // The early read is guaranteed to see a real, non-null trampoline
17472            // image: the fixture has reached its barrier and cannot exec yet.
17473            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17474            assert_eq!(
17475                observe_spawned_image(pid).unwrap().executable,
17476                Some(trampoline)
17477            );
17478            sample.write_all(b"X").unwrap();
17479            let mut child = spawn.await.unwrap();
17480            let early = crate::live_children::read_record(&record).unwrap();
17481            assert_eq!(early.len(), 1);
17482            assert_eq!(early[0].pid, pid);
17483            assert_eq!(
17484                early[0].executable, None,
17485                "unconfirmed trampoline image entered the roster"
17486            );
17487            assert!(child.report_ready.get().is_none());
17488            // The barrier's duration is unrelated to the production five-second
17489            // exec budget. Start the test's confirmation budget upon release.
17490            child.privacy_exec.as_mut().unwrap().deadline =
17491                tokio::time::Instant::now() + Duration::from_secs(30);
17492            exec.write_all(b"X").unwrap();
17493            child.confirm_privacy_exec().await;
17494            assert_eq!(child.spawn_failure, None);
17495            assert!(child.report_ready.get().is_some());
17496            let confirmed = crate::live_children::read_record(&record).unwrap();
17497            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17498            assert_ne!(module, trampoline);
17499            assert_eq!(confirmed[0].executable, Some(module.into()));
17500            child.start_kill().unwrap();
17501            child.wait().await.unwrap();
17502            child.release_roster();
17503        }
17504    }
17505
17506    #[tokio::test]
17507    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17508    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17509        #[cfg(target_os = "macos")]
17510        {
17511            let registry = Arc::new(Registry::default());
17512            let policy = RestartPolicy::new(0, Duration::ZERO);
17513            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
17514            let runtime = supervisor.runtime_config();
17515            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17516            let spec = spec("/bin/sh", &["-c", "exit 121"]);
17517            // Drive spawn and confirmation separately instead of starting the
17518            // monitor. WNOWAIT observes a real exit without consuming its status,
17519            // so confirmation's first try_wait must take the already-exited arm.
17520            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17521            let pid = child.pid;
17522            tokio::task::spawn_blocking(move || {
17523                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17524            })
17525            .await
17526            .unwrap()
17527            .unwrap();
17528            child.privacy_exec.as_mut().unwrap().deadline =
17529                tokio::time::Instant::now() + Duration::from_secs(30);
17530            let status = child.wait().await.unwrap();
17531            assert_eq!(status.code(), Some(121));
17532            assert!(child.privacy_exec.is_none());
17533            assert!(
17534                child.report_ready.get().is_none(),
17535                "an exited module must not publish a live pid"
17536            );
17537            let report = classify_reaped_child_exit(&snapshot, &child, &status);
17538            on_child_exit(
17539                &spec,
17540                policy,
17541                &registry,
17542                &snapshot,
17543                &runtime.terminal_ring,
17544                &runtime.spawn_events,
17545                &runtime.child_roster,
17546                report,
17547                true,
17548            )
17549            .await;
17550            let state = lock_snapshot(&snapshot).unwrap();
17551            assert_eq!(state.state, ModuleState::Failed);
17552            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17553            assert_eq!(state.reported_pid(), None);
17554            drop(state);
17555            let history = runtime.terminal_ring.lock().unwrap().snapshot();
17556            assert_eq!(history.entries.len(), 1);
17557            let terminal = &history.entries[0];
17558            assert_eq!(terminal.exit_code, Some(121));
17559            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17560            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17561            assert_eq!(
17562                terminal.disposition_detail.as_deref(),
17563                Some(policy.budget_exhausted_detail().as_str()),
17564                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17565            );
17566            assert_eq!(child.spawn_failure, None);
17567            child.release_roster();
17568        }
17569    }
17570}
17571
17572/// The daemon's real spawn path hands a subc-wire child its launch nonce on
17573/// descriptor 3, without an environment copy. The shell records the nonce
17574/// and its environment after exec so these tests observe the real handover.
17575#[cfg(all(test, unix))]
17576mod launch_nonce_descriptor_tests {
17577    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17578    use crate::stderr_tail::{StderrRing, StderrTailConfig};
17579    use cortexkit_test_support::ScratchDir;
17580    use std::{
17581        path::PathBuf,
17582        sync::{Arc, Mutex},
17583        time::{Duration, Instant},
17584    };
17585
17586    async fn probe(role: super::SpawnRole) {
17587        let scratch = ScratchDir::new("launch-nonce-descriptor");
17588        let fd_copy = scratch.join("from-descriptor");
17589        let env_copy = scratch.join("environment");
17590        let script = format!(
17591            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17592            fd = fd_copy.display(), env = env_copy.display(),
17593        );
17594        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17595        let spec = ModuleSpec {
17596            module_id: "nonce-descriptor-probe".to_string(),
17597            program: PathBuf::from("/bin/sh"),
17598            args: vec!["-c".to_string(), script],
17599            env: vec![
17600                xdg("XDG_DATA_HOME"),
17601                xdg("XDG_RUNTIME_DIR"),
17602                xdg("XDG_CONFIG_HOME"),
17603                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17604            ],
17605            reserved: true,
17606            reserved_prefixes: Vec::new(),
17607            protocol: ModuleProtocol::Subc,
17608            overlap: Default::default(),
17609        };
17610        let handle = SupervisorHandle::new();
17611        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17612        let roster = ChildRoster::default();
17613        #[cfg(target_os = "macos")]
17614        {
17615            let path = super::test_privacy_trampoline();
17616            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17617        }
17618        let child = super::spawn_child_in_slot(
17619            &spec,
17620            None,
17621            Some(&handle),
17622            &ring,
17623            None,
17624            &roster,
17625            #[cfg(target_os = "linux")]
17626            None,
17627            role,
17628            matches!(role, super::SpawnRole::SwapCandidate),
17629        )
17630        .expect("spawn probe");
17631        let deadline = Instant::now() + Duration::from_secs(10);
17632        while !(fd_copy.exists() && env_copy.exists()) {
17633            assert!(Instant::now() < deadline, "probe never wrote its copies");
17634            tokio::time::sleep(Duration::from_millis(20)).await;
17635        }
17636        let nonce = std::fs::read_to_string(fd_copy).unwrap();
17637        assert!(!nonce.is_empty());
17638        let environment = std::fs::read_to_string(env_copy).unwrap();
17639        assert!(environment
17640            .lines()
17641            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17642        let copy = environment
17643            .lines()
17644            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17645        assert_eq!(
17646            copy, None,
17647            "Unix children must never receive the environment nonce"
17648        );
17649        if matches!(role, super::SpawnRole::Plain) {
17650            assert_eq!(
17651                handle.spawn_nonce(&spec.module_id).as_deref(),
17652                Some(nonce.as_str())
17653            );
17654        }
17655        drop(child);
17656    }
17657
17658    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17659    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17660        probe(super::SpawnRole::Plain).await;
17661    }
17662
17663    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17664    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17665        probe(super::SpawnRole::SwapCandidate).await;
17666    }
17667}
17668
17669#[cfg(all(test, target_os = "linux"))]
17670#[path = "supervise/cgroup_reap_tests.rs"]
17671mod cgroup_reap_tests;
17672
17673#[cfg(all(test, target_os = "linux"))]
17674mod cgroup_containment_tests {
17675    use super::*;
17676    use cortexkit_test_support::ScratchDir;
17677
17678    fn running(pid: u32) -> bool {
17679        // An orphan can remain a zombie until the container init reaps it.
17680        std::fs::read_to_string(format!("/proc/{pid}/stat"))
17681            .ok()
17682            .and_then(|stat| {
17683                stat.rsplit_once(") ")
17684                    .map(|(_, rest)| rest.starts_with('Z'))
17685            })
17686            .is_some_and(|zombie| !zombie)
17687    }
17688
17689    #[tokio::test]
17690    async fn linux_teardown_reaps_the_grandchild() {
17691        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17692    }
17693
17694    #[tokio::test]
17695    async fn linux_shutdown_straggler_reaps_the_grandchild() {
17696        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17697    }
17698
17699    async fn teardown_tree(test_name: &str, shutdown: bool) {
17700        let dir = ScratchDir::new(test_name);
17701        let root = PathBuf::from(format!(
17702            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17703            std::process::id(),
17704            unix_ms_now()
17705        ));
17706        if let Err(error) = std::fs::create_dir(&root) {
17707            assert!(
17708                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17709                "required cgroup test cannot execute: {error}"
17710            );
17711            eprintln!(
17712                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17713                root.display()
17714            );
17715            return;
17716        }
17717        let placement = subc_cgroup::prepare_at(&root)
17718            .expect("prepare isolated kernel cgroup")
17719            .expect("isolated cgroup is delegated");
17720        let module_id = "tree-teardown";
17721        let module = placement
17722            .module_path(module_id)
17723            .expect("create isolated module cgroup");
17724        if !module.join("cgroup.kill").exists() {
17725            std::fs::remove_dir(&module).unwrap();
17726            std::fs::remove_dir(root.join("subc-modules")).unwrap();
17727            std::fs::remove_dir(&root).unwrap();
17728            assert!(
17729                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17730                "required cgroup.kill interface unavailable"
17731            );
17732            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17733            return;
17734        }
17735        let supervisor = Supervisor::new_for_test(
17736            Arc::new(Registry::default()),
17737            RestartPolicy::new(3, Duration::ZERO),
17738        )
17739        .with_cgroup_placement(Some(placement));
17740        let mut runtime = supervisor.runtime_config();
17741        runtime.child_roster = runtime
17742            .child_roster
17743            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17744        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17745        let pid_file = dir.join("grandchild.pid");
17746        let spec = ModuleSpec {
17747            module_id: module_id.to_string(),
17748            program: PathBuf::from("/bin/sh"),
17749            args: vec![
17750                "-c".into(),
17751                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17752                "fixture".into(),
17753                pid_file.display().to_string(),
17754            ],
17755            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17756                .into_iter()
17757                .map(|key| (key.to_string(), dir.display().to_string()))
17758                .collect(),
17759            reserved: false,
17760            reserved_prefixes: Vec::new(),
17761            protocol: ModuleProtocol::None,
17762            overlap: Default::default(),
17763        };
17764        let child =
17765            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17766        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17767        let grandchild: u32 = loop {
17768            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17769                if let Ok(pid) = contents.trim().parse() {
17770                    break pid;
17771                }
17772            }
17773            assert!(
17774                tokio::time::Instant::now() < deadline,
17775                "grandchild pid was not recorded"
17776            );
17777            tokio::time::sleep(Duration::from_millis(10)).await;
17778        };
17779        assert!(
17780            running(grandchild),
17781            "grandchild must be alive before teardown"
17782        );
17783        if shutdown {
17784            let mut child = child;
17785            crate::child_roster::end_children_for_daemon_shutdown(
17786                &runtime.child_roster,
17787                false,
17788                std::future::pending(),
17789            )
17790            .await;
17791            child.wait().await.expect("reap shutdown straggler");
17792        } else {
17793            drain_child_to_state(
17794                module_id,
17795                ModuleProtocol::None,
17796                StopNotice::NotSent,
17797                &Registry::default(),
17798                None,
17799                &snapshot,
17800                &runtime.terminal_ring,
17801                &SpawnEventFeed::default(),
17802                child,
17803                Duration::from_millis(100),
17804                ModuleState::Stopped,
17805                Some(false),
17806                &FailureContext::operator(),
17807            )
17808            .await
17809            .expect("real supervisor teardown");
17810        }
17811        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17812        while running(grandchild) && tokio::time::Instant::now() < deadline {
17813            tokio::time::sleep(Duration::from_millis(10)).await;
17814        }
17815        let survived = running(grandchild);
17816        // Kill a surviving grandchild so a failed test does not leave it behind.
17817        if survived {
17818            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17819            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17820            tokio::time::sleep(Duration::from_millis(100)).await;
17821        }
17822        if module.exists() {
17823            std::fs::remove_dir(&module).expect("remove empty module cgroup");
17824        }
17825        std::fs::remove_dir(root.join("subc-modules")).unwrap();
17826        std::fs::remove_dir(&root).unwrap();
17827        assert!(
17828            !survived,
17829            "grandchild {grandchild} outlived module teardown"
17830        );
17831        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17832    }
17833}
17834
17835/// A slow live-children record write on the restart path must not stop the
17836/// daemon from answering the replacement module's authentication handshake.
17837#[cfg(all(test, unix))]
17838mod record_write_stall_tests {
17839    use super::*;
17840    use std::sync::atomic::{AtomicBool, Ordering};
17841    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17842
17843    /// `subc-client-rs` gives the whole handshake this long before the module
17844    /// exits with `Timeout { stage: ServerProof }`.
17845    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17846    /// How long the replacement's admission write is held: longer than the
17847    /// module's deadline, standing in for the delay a busy disk can add
17848    /// across the two syncs in that write.
17849    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17850
17851    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
17852    /// On macOS every module is launched through the privacy trampoline, and
17853    /// a module that is not a `#!` script is admitted only if its running
17854    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
17855    /// the shell soon replaces itself with `/bin/sleep`, which does not match
17856    /// the configured `/bin/sh`. The image is read only after the spawn's
17857    /// live-children record write, and here that write lasted long enough for
17858    /// the shell to have exec'd every time, so each process was refused and
17859    /// killed right after spawn, and the supervisor crash-looped it with
17860    /// backoff until its restart budget ran out. The restart this test
17861    /// measures would then never produce a module at all.
17862    fn sleeping_spec() -> ModuleSpec {
17863        ModuleSpec {
17864            module_id: "record-write-stall".into(),
17865            program: "/bin/sleep".into(),
17866            args: vec!["600".into()],
17867            env: vec![],
17868            reserved: false,
17869            reserved_prefixes: vec![],
17870            protocol: ModuleProtocol::None,
17871            overlap: Default::default(),
17872        }
17873    }
17874
17875    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17876    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17877        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17878        let supervisor =
17879            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17880                .with_live_children_record(home.join("live-children.json"));
17881        let module = supervisor.spawn(sleeping_spec()).unwrap();
17882        // `reported_pid` names a process only once it is confirmed as the
17883        // module (on macOS, once the trampoline's image check has passed), so
17884        // a process the daemon is about to refuse and kill never counts as
17885        // the running module, here or in the final wait below.
17886        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17887            loop {
17888                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17889                    .unwrap()
17890                    .reported_pid()
17891                {
17892                    return pid;
17893                }
17894                tokio::time::sleep(Duration::from_millis(10)).await;
17895            }
17896        })
17897        .await
17898        .expect("the first process was admitted as the module");
17899
17900        // The daemon's listener, served by this runtime's workers like the
17901        // supervisor's own tasks.
17902        let key = vec![0x42; 32];
17903        let daemon_id = [0x24; 16];
17904        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17905        let port = listener.local_addr().unwrap().port();
17906        let server = tokio::spawn(crate::server::serve_listener(
17907            listener,
17908            Arc::new(crate::router::Router::with_default_self_handler()),
17909            crate::server::ServerAuth::with_limits(
17910                key.clone(),
17911                daemon_id,
17912                "test",
17913                crate::server::DEFAULT_AUTH_DEADLINE,
17914                4,
17915            ),
17916        ));
17917
17918        // Hold the write that admits the replacement, once, and say when it
17919        // has started.
17920        let (held_tx, held_rx) = std::sync::mpsc::channel();
17921        let armed = AtomicBool::new(true);
17922        supervisor
17923            .child_roster
17924            .set_record_write_hook_for_test(move |live| {
17925                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17926                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17927                    let _ = held_tx.send(());
17928                    std::thread::sleep(HELD_RECORD_WRITE);
17929                }
17930            });
17931
17932        // The replacement module, as a separate process would be: its own
17933        // thread and runtime, so only the daemon's runtime can be stalled.
17934        let module_process = std::thread::spawn(move || {
17935            held_rx
17936                .recv_timeout(Duration::from_secs(30))
17937                .expect("the restart reached the replacement's record write");
17938            let conn = ConnectionInfo {
17939                schema: SCHEMA_VERSION,
17940                wire_version: None,
17941                endpoints: vec![Endpoint {
17942                    host: "127.0.0.1".to_owned(),
17943                    port,
17944                }],
17945                key,
17946                daemon_id,
17947                pid: std::process::id(),
17948                daemon_ver: "test".to_owned(),
17949            };
17950            tokio::runtime::Builder::new_current_thread()
17951                .enable_all()
17952                .build()
17953                .unwrap()
17954                .block_on(async {
17955                    let started = Instant::now();
17956                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17957                        .await
17958                        .unwrap();
17959                    let result =
17960                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17961                    (result, started.elapsed())
17962                })
17963        });
17964
17965        module.restart(Some(0)).await.unwrap();
17966        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17967            .await
17968            .unwrap();
17969        assert!(
17970            auth.is_ok(),
17971            "the daemon must answer a module's handshake while a restart waits on the \
17972             live-children record write; got {auth:?} after {elapsed:?}"
17973        );
17974
17975        tokio::time::timeout(Duration::from_secs(20), async {
17976            while lock_snapshot(&module.inner.snapshot)
17977                .unwrap()
17978                .reported_pid()
17979                .is_none_or(|pid| pid == old_pid)
17980            {
17981                tokio::time::sleep(Duration::from_millis(10)).await;
17982            }
17983        })
17984        .await
17985        .expect("the restart published its replacement");
17986        module.stop().await.unwrap();
17987        server.abort();
17988    }
17989}