Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68/// Command-line flag used by supervised modules to find subc.
69///
70/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
71/// The path points at the TCP+key connection file; it is not an ambient signal and
72/// is never inherited by standalone children.
73pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
79/// to contain a real crash loop (which respawns in seconds) and short enough
80/// that unrelated crashes hours apart never accumulate into a permanent stop.
81const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82/// How long a drain waits for already-dispatched requests to finalize before
83/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
84/// not RPC-scale: the original 2s value silently cut nearly every real tool
85/// call at the fence, making the wait-for-finalize design decorative for the
86/// workloads it existed for. Quiescence short-circuits, so an idle module
87/// restarts immediately regardless of this value; the budget is spent only
88/// when a genuine in-flight request is worth finishing. Per-module override:
89/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
90/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
91/// where a stuck request will never settle).
92pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96/// How long a drain waits after one busy-gauge probe completes before it asks
97/// again.
98///
99/// A drain re-probes on this short interval of its own rather than on the
100/// module's health cadence. The default cadence (30 s) equals the default
101/// drain ceiling, so a cadence-paced drain that found the module busy at its
102/// first probe never asked a second time: it waited out the whole ceiling and
103/// forced GOODBYE even after the gauges had read zero. The interval is counted
104/// from the moment the previous probe finished, and probes run one at a time,
105/// so a slow module is never asked more often than it can answer. Periodic
106/// health checks outside a drain keep using the configured cadence.
107const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108/// How long a restart waits for the exited process's output readers.
109///
110/// The restart does not depend on either output reader finishing. A stderr
111/// reader still running at this bound is left running, and its later output goes
112/// into the exited process's own section of the stderr ring (see
113/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
114/// bound no longer decides whether a crash's last lines are kept: under load
115/// the reader may simply not have been scheduled yet, and cutting it there
116/// discarded exactly the lines that explained the crash.
117///
118/// What the bound still decides is when the tail starts reporting
119/// `Incomplete`: a pipe open past it usually means a descendant of the exited
120/// process still holds it, and the tail cannot claim to be whole until that
121/// pipe closes. That is also the only case in which the wait costs the restart
122/// anything, because a pipe with no other holder reaches EOF when the process
123/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
124/// `Captured` at EOF with nothing lost.
125///
126/// The stdout reader carries no ring, only the capture file. It also continues
127/// to EOF after this bound: child exit closes the writer but does not mean the
128/// reader has consumed all buffered bytes. Late output from either pipe can
129/// therefore reach the shared capture file after the successor starts.
130const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
132pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
135/// dropped because the subscriber stopped draining its frames. The detail's
136/// `first_undelivered_cursor` names the first event it did not receive; the
137/// client resubscribes from the last cursor it did receive.
138pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143    deadline: tokio::time::Instant,
144    expected: Option<subc_os::FileIdentity>,
145    trampoline: Option<subc_os::FileIdentity>,
146    script: bool,
147    module_id: String,
148    #[cfg(test)]
149    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
155    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
156    let path = std::env::current_exe()
157        .unwrap()
158        .parent()
159        .unwrap()
160        .parent()
161        .unwrap()
162        .join("privacy-trampoline-fixture");
163    // Without the fixture every macOS spawn is refused, and the tests that
164    // spawn fail later as a module in state Failed, which names the wrong
165    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
166    assert!(
167        path.exists(),
168        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169         --bins --features test-support` or `cargo test -p subc-daemon` first",
170        path.display()
171    );
172    path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177    use std::io::Read;
178    let mut probe = std::process::Command::new(path)
179        .args(["__disclaim-exec", "--probe"])
180        .stdin(Stdio::null())
181        .stdout(Stdio::piped())
182        .stderr(Stdio::piped())
183        .spawn()
184        .map_err(|error| {
185            format!(
186                "privacy trampoline probe failed for {}: {error}",
187                path.display()
188            )
189        })?;
190    let deadline = std::time::Instant::now() + Duration::from_secs(5);
191    let status = loop {
192        match probe.try_wait() {
193            Ok(Some(status)) => break status,
194            Ok(None) if std::time::Instant::now() < deadline => {
195                std::thread::sleep(Duration::from_millis(5))
196            }
197            result => {
198                let _ = probe.kill();
199                let _ = probe.wait();
200                return Err(format!(
201                    "privacy trampoline probe failed or timed out for {}: {result:?}",
202                    path.display()
203                ));
204            }
205        }
206    };
207    let mut answer = String::new();
208    if let Some(stdout) = probe.stdout.take() {
209        let _ = stdout.take(256).read_to_string(&mut answer);
210    }
211    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212        return Ok(());
213    }
214    let mut diagnostic = String::new();
215    if let Some(stderr) = probe.stderr.take() {
216        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217    }
218    let cause = diagnostic
219        .trim()
220        .strip_prefix("ck-subc: own privacy identity refused: ")
221        .unwrap_or("binary does not implement the privacy trampoline protocol");
222    Err(format!(
223        "{cause}: probe of {} exited {status}",
224        path.display()
225    ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230    spec: &ModuleSpec,
231    roster: &ChildRoster,
232) -> Result<
233    (
234        Command,
235        Option<PrivacyExec>,
236        subc_os::privacy_identity::ExecAcknowledgement,
237    ),
238    SuperviseError,
239> {
240    let failure = |cause: String| {
241        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242        SuperviseError::Spawn {
243            program: spec.program.clone(),
244            source: io::Error::other(cause),
245            cgroup_path: None,
246        }
247    };
248    let trampoline = roster.privacy_trampoline().map_err(failure)?;
249    // Resolve PATH with the same environment the Command will receive. For
250    // scripts retain the existing orphan-identity rule: the kernel chooses
251    // the interpreter, and its observed image is the one recorded. Do not
252    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
253    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254        let path = spec
255            .env
256            .iter()
257            .find(|(key, _)| key == "PATH")
258            .map(|(_, value)| std::ffi::OsString::from(value))
259            .or_else(|| std::env::var_os("PATH"))
260            .unwrap_or_else(|| "/usr/bin:/bin".into());
261        std::env::split_paths(&path)
262            .map(|dir| dir.join(&spec.program))
263            .find(|path| path.is_file())
264            .unwrap_or_else(|| spec.program.clone())
265    } else {
266        spec.program.clone()
267    };
268    let expected = subc_os::file_identity(&program);
269    let trampoline_image = subc_os::file_identity(&trampoline);
270    let script = {
271        use std::io::Read;
272        let mut prefix = [0u8; 2];
273        std::fs::File::open(&program)
274            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275    };
276    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277        return Err(failure(
278            "privacy identity module executable is missing or is the trampoline itself".to_string(),
279        ));
280    }
281    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282        .map_err(|error| failure(error.to_string()))?;
283    let reader =
284        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285    let mut command = Command::new(&trampoline);
286    command
287        .arg("__disclaim-exec")
288        .arg(ack.fd().to_string())
289        .arg(&program);
290    ack.install(command.as_std_mut());
291    Ok((
292        command,
293        Some(PrivacyExec {
294            reader,
295            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296            expected,
297            trampoline: trampoline_image,
298            script,
299            module_id: spec.module_id.clone(),
300            #[cfg(test)]
301            confirmation_pause: spec
302                .env
303                .iter()
304                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305                .map(|(_, address)| (address.clone(), None)),
306        }),
307        ack,
308    ))
309}
310
311struct SupervisedChild {
312    child: Child,
313    exec: crate::activity::ExecOwner,
314    #[cfg(windows)]
315    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
316    #[cfg(target_os = "macos")]
317    privacy_exec: Option<PrivacyExec>,
318    /// Set once this launch's exec acknowledgement confirms the module image.
319    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
320    /// `subc_os::privacy_identity`), which then replaces itself with the
321    /// module. The supervisor owns and can kill that pid from spawn, but
322    /// status readers report it only after this latch is set, so nothing
323    /// reports the trampoline's image as the module's.
324    #[cfg(target_os = "macos")]
325    report_ready: Arc<OnceLock<()>>,
326    /// Supervisor failure detail retained for the terminal record, including
327    /// image refusals and first-HELLO timeouts after a successful spawn.
328    spawn_failure: Option<String>,
329    /// The protocol this process was launched with. A reload can store a new
330    /// launch spec with a different protocol, but that takes effect only at the
331    /// next spawn, so this process keeps being handled by the protocol it
332    /// actually speaks.
333    protocol: ModuleProtocol,
334    /// This process's cgroup name: a bounded module/slot label followed by a
335    /// spawn suffix unique to this process (when cgroup placement is on). A
336    /// retired process in a slot may still be draining when a later one is
337    /// spawned into that slot, so the suffix keeps the later process out of
338    /// the retired one's cgroup, which is the domain a kill applies to.
339    #[cfg(target_os = "linux")]
340    module_id: String,
341    #[cfg(target_os = "linux")]
342    cgroup_placement: Option<subc_cgroup::Placement>,
343    /// The job that contains this child and every process it spawns (issue #109).
344    ///
345    /// Dropping this handle is what reaps a surviving tree when no supervisor
346    /// code runs — a daemon crash — because the job carries
347    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
348    ///
349    /// That limit is not crash-only, and the difference is worth knowing: a
350    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
351    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
352    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
353    /// module at once. Before this change they survived that, saw EOF on the
354    /// control socket, and ran their own teardown; Unix keeps that path
355    /// deliberately, so a module can seal a WAL or close a capture rather than
356    /// be killed mid-write. So this trades graceful teardown on every Windows
357    /// daemon stop for containment on a crash, which is the right way round
358    /// today: orphaned GPU workers are a reported, recurring problem, and the
359    /// modules that write most heavily do not run on Windows.
360    ///
361    /// The fix is a real Windows stop path — the daemon draining before it
362    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
363    /// reaches only what the drain left behind, which is what it should reach.
364    #[cfg(windows)]
365    job: Option<subc_jobobject::JobObject>,
366    stdout_pump: Option<JoinHandle<()>>,
367    stderr_pump: Option<StderrPump>,
368    stderr_ring: Arc<Mutex<StderrRing>>,
369    /// Conservative monotonic bound sampled before `command.spawn()`, not when
370    /// identity recording finishes or the spawn generation is published.
371    spawned_at: Instant,
372    /// Ordinary spawns must send their first HELLO within this budget. Once
373    /// accepted, a later disconnect must not restart the registration clock.
374    registration_deadline: Option<Instant>,
375    registration_watch: Option<RegistrationWatch>,
376    spawned_at_ms: u64,
377    spawned_from: PathBuf,
378    launch_spec_digest: LaunchSpecDigest,
379    spawned_file_identity: Option<SpawnedFileIdentity>,
380    process_start_time: Option<u64>,
381    process_identity: Option<ProcessIdentity>,
382    pid: u32,
383    /// This process's entry in the daemon's child roster, released when the
384    /// process is reaped or this handle is dropped.
385    roster_guard: Option<crate::child_roster::RosterGuard>,
386}
387
388impl SupervisedChild {
389    fn registered_since_spawn(&self) -> bool {
390        self.registration_watch
391            .as_ref()
392            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
393    }
394
395    fn id(&self) -> Option<u32> {
396        Some(self.pid)
397    }
398
399    fn process_identity(&self) -> Option<ProcessIdentity> {
400        self.process_identity
401    }
402
403    async fn wait(&mut self) -> io::Result<ExitStatus> {
404        #[cfg(target_os = "macos")]
405        self.confirm_privacy_exec().await;
406        // The roster entry is NOT released here. A daemon shutdown waits for the
407        // roster to empty and then exits the process, so releasing at the reap
408        // let it exit before the exit handler wrote this child's terminal record
409        // (the stderr drain and snapshot update sit in between), and the
410        // shutdown's own `daemon_shutdown` record was intermittently lost. The
411        // caller releases it after recording the exit (`release_roster`), and
412        // dropping the handle releases it too.
413        let result = self.child.wait().await;
414        #[cfg(target_os = "linux")]
415        if result.is_ok() {
416            if let Some(placement) = self.cgroup_placement.as_ref() {
417                cleanup_reaped_cgroup(placement, &self.module_id).await;
418                // Keep ownership while awaiting kernel population changes: a
419                // drain timeout may cancel this wait and then escalate/reap.
420                self.cgroup_placement = None;
421            }
422        }
423        result
424    }
425
426    #[cfg(target_os = "macos")]
427    async fn confirm_privacy_exec(&mut self) {
428        let Some(pending) = &mut self.privacy_exec else {
429            return;
430        };
431        let result = tokio::time::timeout_at(pending.deadline, async {
432            let mut record = Vec::new();
433            loop {
434                let mut ready = pending.reader.readable().await?;
435                let read = ready.try_io(|reader| {
436                    use std::io::Read;
437                    let mut reader = reader.get_ref();
438                    let mut buffer = [0u8; 256];
439                    reader.read(&mut buffer).map(|count| (count, buffer))
440                });
441                match read {
442                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
443                    Ok(Ok((count, buffer))) => {
444                        if record.len() + count > 1024 {
445                            return Err(io::Error::other(
446                                "privacy exec refusal record is too long",
447                            ));
448                        }
449                        record.extend_from_slice(&buffer[..count]);
450                    }
451                    Ok(Err(error)) => return Err(error),
452                    Err(_) => continue,
453                }
454            }
455        })
456        .await;
457        #[cfg(test)]
458        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
459            use tokio::io::{AsyncReadExt, AsyncWriteExt};
460            if stream.is_none() {
461                *stream = Some(
462                    tokio::net::TcpStream::connect(address.as_str())
463                        .await
464                        .unwrap(),
465                );
466                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
467                stream
468                    .as_mut()
469                    .unwrap()
470                    .write_all(&self.pid.to_ne_bytes())
471                    .await
472                    .unwrap();
473            }
474            let mut reply = [0];
475            stream
476                .as_mut()
477                .unwrap()
478                .read_exact(&mut reply)
479                .await
480                .unwrap();
481            pending.confirmation_pause = None;
482            if &reply == b"F" {
483                Ok(Err(io::Error::other("injected late exec refusal")))
484            } else {
485                assert_eq!(&reply, b"X");
486                result
487            }
488        } else {
489            result
490        };
491        // Keep the reader in self across await: select cancellation must not
492        // discard the handshake or reset its original five-second deadline.
493        let pending = self.privacy_exec.as_ref().expect("pending exec");
494        let cause = match result {
495            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
496            Ok(Err(error)) => Some(format!(
497                "privacy identity exec acknowledgement failed: {error}"
498            )),
499            Ok(Ok(record)) if !record.is_empty() => Some(
500                std::str::from_utf8(&record)
501                    .ok()
502                    .and_then(|record| {
503                        record
504                            .trim()
505                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
506                    })
507                    .filter(|cause| !cause.is_empty())
508                    .unwrap_or("invalid privacy exec refusal record")
509                    .to_string(),
510            ),
511            Ok(Ok(_)) => match self.child.try_wait() {
512                // Empty EOF is the exec acknowledgement. A real module may exit
513                // immediately, including with a reserved trampoline status; no
514                // image is admitted, and its ordinary exit contract stays intact.
515                Ok(Some(_status)) => None,
516                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
517                Ok(None) => {
518                    let image = observe_spawned_image(self.pid);
519                    if let Some(image) = image.filter(|image| {
520                        image.executable.is_some()
521                            && image.executable != pending.trampoline
522                            && (image.executable == pending.expected || pending.script)
523                    }) {
524                        if let Err(error) = self
525                            .roster_guard
526                            .as_ref()
527                            .map(|guard| guard.confirm_executable(image))
528                            .transpose()
529                        {
530                            Some(format!(
531                                "could not durably record confirmed module image: {error}"
532                            ))
533                        } else {
534                            let _ = self.report_ready.set(());
535                            info!(module_id = %pending.module_id, pid = self.pid,
536                                "module spawned with own privacy identity (responsibility disclaimed)");
537                            None
538                        }
539                    } else if image.is_none()
540                        || image.is_some_and(|image| image.executable.is_none())
541                    {
542                        // A process can exit between try_wait and the kernel
543                        // image read. Empty EOF already acknowledged exec, so
544                        // preserve that module's ordinary exit rather than
545                        // mislabel a disappearing image as trampoline refusal.
546                        // Keep pending in self across await so cancellation does
547                        // not discard validation or reset its original deadline.
548                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
549                            Ok(Ok(_status)) => None,
550                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
551                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
552                        }
553                    } else {
554                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
555                    }
556                }
557            },
558        };
559        let pending = self.privacy_exec.take().expect("pending exec");
560        // Exec confirmation describes launch success, not roster retirement.
561        // Keep even a reaped child recorded until its terminal history exists.
562        self.exec.0.finish(if cause.is_some() {
563            crate::activity::ExecState::Refused
564        } else {
565            crate::activity::ExecState::CommittedExec
566        });
567        if let Some(cause) = cause {
568            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
569            self.spawn_failure = Some(cause);
570            // No image is admitted on failure. Reach the entire fresh process
571            // group, including a module which spawned a helper before refusal.
572            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
573                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
574            }
575            let _ = self.child.start_kill();
576        }
577    }
578
579    /// Releases this child's daemon-shutdown roster entry once its exit has
580    /// been recorded. The pid is already reaped and free for reuse, so the
581    /// entry must not outlive the record any longer than that.
582    fn release_roster(&mut self) {
583        self.roster_guard = None;
584    }
585
586    /// Kill the child and, where containment is available, its process tree.
587    ///
588    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
589    /// helper process leaked the helper — the Synapse embedding module's CUDA
590    /// worker holds the GPU allocation, so the leak cost VRAM until the next
591    /// restart of something else. Terminating the job reaches grandchildren that
592    /// a tree walk cannot, including one whose parent has already exited and
593    /// been reparented away.
594    ///
595    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
596    /// direct-child kill still decides the outcome, so containment can never
597    /// change whether a module is reported as stopped.
598    fn start_kill(&mut self) -> io::Result<()> {
599        #[cfg(windows)]
600        if let Some(job) = &self.job {
601            if let Err(error) = job.terminate() {
602                debug!(
603                    error = %error,
604                    "job termination failed; the direct-child kill still owns the outcome"
605                );
606            }
607        }
608        #[cfg(target_os = "linux")]
609        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
610        self.child.start_kill()
611    }
612
613    async fn drain_stderr(&mut self, module_id: &str) {
614        if let Some(pump) = self.stdout_pump.take() {
615            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
616        }
617
618        let Some(pump) = self.stderr_pump.take() else {
619            return;
620        };
621        settle_stderr_pump(
622            module_id,
623            &self.stderr_ring,
624            pump,
625            STDERR_PUMP_DRAIN_TIMEOUT,
626        )
627        .await;
628    }
629}
630
631/// Bound the restart's wait, not the lifetime of the exited child's stdout
632/// reader. Slow pipe reads or capture writes must not turn the bound into lost
633/// output; dropping the handle lets the reader finish at EOF, like stderr.
634async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
635    match timeout(bound, &mut pump).await {
636        Ok(Ok(())) => {}
637        Ok(Err(error)) => {
638            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
639        }
640        Err(_) => {
641            drop(pump);
642            warn!(
643                module_id,
644                waited = ?bound,
645                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
646            );
647        }
648    }
649}
650
651/// The reader task for one process's stderr, with the ring generation its
652/// lines are attributed to.
653struct StderrPump {
654    task: JoinHandle<()>,
655    generation: u64,
656}
657
658/// Retire an exited process's stderr reader and wait up to `bound` for it to
659/// reach EOF. A reader still running at the bound is detached, not stopped: it
660/// keeps filling the exited process's section of the ring until its pipe
661/// closes, and the tail reads `Incomplete` until then. See
662/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
663async fn settle_stderr_pump(
664    module_id: &str,
665    ring: &Arc<Mutex<StderrRing>>,
666    pump: StderrPump,
667    bound: Duration,
668) {
669    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
670    let StderrPump {
671        mut task,
672        generation,
673    } = pump;
674    lock().retire_pump(generation);
675    match timeout(bound, &mut task).await {
676        Ok(Ok(())) => {}
677        Ok(Err(err)) => {
678            let mut ring = lock();
679            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
680            ring.finish_pump(generation);
681            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
682        }
683        Err(_) => {
684            // Dropping the handle detaches the task; it ends at EOF on its pipe.
685            drop(task);
686            lock().mark_pump_late(
687                generation,
688                format!(
689                    "stderr of the exited process had not reached EOF {bound:?} after it was \
690                     retired (a descendant may still hold the pipe open); lines it still \
691                     writes are kept in that process's section"
692                ),
693            );
694            warn!(
695                module_id,
696                waited = ?bound,
697                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
698            );
699        }
700    }
701}
702
703fn registration_release_events() -> &'static watch::Sender<u64> {
704    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
705    EVENTS.get_or_init(|| {
706        let (sender, _receiver) = watch::channel(0);
707        sender
708    })
709}
710
711pub(crate) fn notify_registration_release() {
712    let events = registration_release_events();
713    let next_generation = (*events.borrow()).wrapping_add(1);
714    events.send_replace(next_generation);
715}
716
717/// How to launch one singleton module process.
718#[derive(Debug, Clone, PartialEq, Eq)]
719pub struct ModuleSpec {
720    pub module_id: String,
721    pub program: PathBuf,
722    pub args: Vec<String>,
723    pub env: Vec<(String, String)>,
724    /// When true this is a reserved module: each spawn gets a fresh one-time launch
725    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
726    /// process can register this module_id (a security-boundary module like the
727    /// credential vault must not be impersonable while it is down/restarting).
728    pub reserved: bool,
729    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
730    /// Prefixes come from daemon config and must end in `:` before they reach the
731    /// supervisor; the owner module's current spawn nonce authorizes claims under
732    /// each prefix.
733    pub reserved_prefixes: Vec<String>,
734    /// The wire protocol this module speaks, as DECLARED in daemon config.
735    ///
736    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
737    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
738    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
739    /// and NO launch nonce, and a clean exit the daemon did not request is
740    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
741    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
742    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
743    /// because a process ignores an environment variable it does not read.
744    ///
745    /// The argument is the part that cannot be "harmless to a process that
746    /// ignores it": a stock binary exits on an unknown flag before it listens
747    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
748    /// first conformance run against this mode found it. The nonce is withheld
749    /// because a process that will never present it gains nothing from holding
750    /// it, and a secret in the environment of a process that does not need it is
751    /// a leak surface for no benefit.
752    pub protocol: ModuleProtocol,
753    /// Whether two processes of this module may run at once, which is what a
754    /// blue/green swap does for the length of its overlap. Declared in daemon
755    /// config because the daemon must be able to answer it while the module is
756    /// down, and so a module cannot talk itself into it after registering.
757    pub overlap: ModuleOverlap,
758}
759
760/// Only the configured launch inputs enter this value; spawned connection paths,
761/// nonces and inherited environment never do. Its bytes remain daemon-private.
762#[derive(Clone, PartialEq, Eq)]
763pub(crate) struct LaunchSpecDigest([u8; 32]);
764
765impl fmt::Debug for LaunchSpecDigest {
766    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
767        f.write_str("LaunchSpecDigest([redacted])")
768    }
769}
770
771impl LaunchSpecDigest {
772    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
773        fn field(hash: &mut Sha256, bytes: &[u8]) {
774            hash.update((bytes.len() as u64).to_le_bytes());
775            hash.update(bytes);
776        }
777        let mut hash = Sha256::new();
778        if let Some(program) = spec.program.to_str() {
779            field(&mut hash, b"utf8-program");
780            field(&mut hash, program.as_bytes());
781        } else {
782            // The daemon config is UTF-8, but in-process callers may provide
783            // native paths: lossy conversion would collapse distinct programs.
784            #[cfg(unix)]
785            {
786                use std::os::unix::ffi::OsStrExt;
787                field(&mut hash, b"unix-program");
788                field(&mut hash, spec.program.as_os_str().as_bytes());
789            }
790            #[cfg(windows)]
791            {
792                use std::os::windows::ffi::OsStrExt;
793                field(&mut hash, b"windows-program");
794                let bytes = spec
795                    .program
796                    .as_os_str()
797                    .encode_wide()
798                    .flat_map(u16::to_le_bytes)
799                    .collect::<Vec<_>>();
800                field(&mut hash, &bytes);
801            }
802            #[cfg(not(any(unix, windows)))]
803            field(&mut hash, spec.program.to_string_lossy().as_bytes());
804        }
805        hash.update((spec.args.len() as u64).to_le_bytes());
806        for arg in &spec.args {
807            field(&mut hash, arg.as_bytes());
808        }
809        let mut env = spec.env.iter().collect::<Vec<_>>();
810        env.sort_unstable();
811        hash.update((env.len() as u64).to_le_bytes());
812        for (key, value) in env {
813            field(&mut hash, key.as_bytes());
814            field(&mut hash, value.as_bytes());
815        }
816        field(
817            &mut hash,
818            match spec.protocol {
819                ModuleProtocol::Subc => b"subc",
820                ModuleProtocol::None => b"none",
821            },
822        );
823        Self(hash.finalize().into())
824    }
825}
826
827/// Whether a module tolerates a second process of itself running alongside.
828///
829/// Most modules are single-writer on their store (a WAL, a capture log, a
830/// resident index behind a writer barrier), and two processes on one store
831/// corrupt it. So a swap, which overlaps the old and new process by design,
832/// is refused unless the module's config opts in with `overlap: "safe"`.
833#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
834pub enum ModuleOverlap {
835    /// Never run two processes of this module at once. The default.
836    #[default]
837    Exclusive,
838    /// The module has said a second process of itself is harmless for the
839    /// length of a swap.
840    ///
841    /// Declare it only if a second instance can run for a few seconds without
842    /// touching ANY single-writer store: every database, WAL, index, projector
843    /// and scheduled job the module owns. A lease on part of that state is not
844    /// enough. broca's session lease guards WAL appends while its run index, its
845    /// store projector and its archive fold timer (which unlinks live WAL files)
846    /// stay single-writer, so broca is exclusive despite holding a lease. The
847    /// refusal only fires after this has been decided, so the decision is the
848    /// check.
849    Safe,
850}
851
852impl ModuleOverlap {
853    pub fn as_str(self) -> &'static str {
854        match self {
855            Self::Exclusive => "exclusive",
856            Self::Safe => "safe",
857        }
858    }
859}
860
861/// Environment variable telling a spawned module which case it was started
862/// for, before it sends HELLO. Only a swap candidate carries it, as
863/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
864///
865/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
866/// longer because nobody waits on it, while a plain restart must flip ready
867/// quickly because callers see `module_warming` until it does. Absence means
868/// plain restart, the safe reading. The daemon trusts nothing about it; the
869/// candidate is proven by its launch nonce at HELLO.
870pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
871/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
872pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
873/// How long a swap waits for its candidate to register and declare itself
874/// ready when the operator does not say. A module warming as a swap candidate
875/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
876/// daemon allows that plus time to start the process and send HELLO.
877pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
878
879/// Bounded restart policy for crash exits.
880///
881/// `max_restarts` is the number of replacement processes allowed after the
882/// initial spawn WITHIN `window`. After that many crash restarts inside one
883/// window the module enters [`ModuleState::Failed`] and the supervisor stops
884/// the crash loop.
885///
886/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
887/// and that only survived because crashes were rare: a module that crashed
888/// three times across a week was disabled forever by crashes that had nothing
889/// to do with each other. That stopped being survivable once modules began
890/// exiting non-zero whenever the daemon's connection to them drops, because
891/// then every daemon-side connection drop spends a unit of the same budget and
892/// one flappy hour permanently stops a healthy module. Restarts older than
893/// `window` release their slot, so a module that crashed twice yesterday has a
894/// full budget today, while a genuine crash loop -- which is fast by
895/// definition -- still reaches the cap and stops.
896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
897pub struct RestartPolicy {
898    pub max_restarts: u32,
899    /// Base delay before a crash replacement. The actual delay escalates with
900    /// the number of recent crash replacements and is capped by `max_backoff`.
901    pub backoff: Duration,
902    /// Maximum delay before a crash replacement.
903    pub max_backoff: Duration,
904    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
905    /// budget effectively infinite (nothing is ever in-window), which is why
906    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
907    pub window: Duration,
908}
909
910impl RestartPolicy {
911    /// A policy with the default crash window. Callers that care about the
912    /// window say so with [`Self::with_window`]; the ones that do not are
913    /// asking for the standard rate limit, not for no limit.
914    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
915        Self {
916            max_restarts,
917            backoff,
918            max_backoff: DEFAULT_MAX_BACKOFF,
919            window: DEFAULT_RESTART_WINDOW,
920        }
921    }
922
923    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
924        self.max_backoff = max_backoff;
925        self
926    }
927
928    pub fn with_window(mut self, window: Duration) -> Self {
929        self.window = window;
930        self
931    }
932
933    /// Calculate the capped exponential delay for the next crash replacement.
934    /// `restart_in_window` is zero for the first replacement after an operator
935    /// action (restart, reload, re-enable) cleared the crash ring, or after all
936    /// older crash replacements have aged out of the window.
937    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
938        if self.backoff.is_zero() || self.max_backoff.is_zero() {
939            return Duration::ZERO;
940        }
941
942        let mut delay = self.backoff;
943        for _ in 0..restart_in_window {
944            if delay >= self.max_backoff {
945                return self.max_backoff;
946            }
947            delay = delay
948                .checked_mul(10)
949                .unwrap_or(self.max_backoff)
950                .min(self.max_backoff);
951        }
952        delay.min(self.max_backoff)
953    }
954
955    /// The one sentence that explains a budget-exhausted stop, used for both the
956    /// log line and the terminal record so the two cannot drift. It names the
957    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
958    /// exactly what this budget is not.
959    fn budget_exhausted_detail(&self) -> String {
960        format!(
961            "crash budget exhausted: max_restarts={} within window_secs={}",
962            self.max_restarts,
963            self.window.as_secs()
964        )
965    }
966}
967
968impl Default for RestartPolicy {
969    fn default() -> Self {
970        Self {
971            max_restarts: DEFAULT_MAX_RESTARTS,
972            backoff: DEFAULT_BACKOFF,
973            max_backoff: DEFAULT_MAX_BACKOFF,
974            window: DEFAULT_RESTART_WINDOW,
975        }
976    }
977}
978
979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
980struct CrashRestartSchedule {
981    restart_in_window: u32,
982    delay: Duration,
983}
984
985/// Whether the daemon itself will bring this module back after the exit being
986/// handled: it is enabled AND its in-window crash restarts are below the cap.
987///
988/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
989/// the window are dropped here rather than by a timer, so the count is right
990/// the moment somebody asks and no bookkeeping runs for idle modules.
991fn daemon_will_restart(
992    state: &mut SupervisorSnapshot,
993    policy: &RestartPolicy,
994    now: Instant,
995) -> bool {
996    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
997}
998
999const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1000const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1001const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1002const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1003
1004#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1005pub enum HealthAction {
1006    Report,
1007    Restart,
1008    Alert,
1009}
1010
1011impl fmt::Display for HealthAction {
1012    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1013        f.write_str(match self {
1014            Self::Report => "report",
1015            Self::Restart => "restart",
1016            Self::Alert => "alert",
1017        })
1018    }
1019}
1020
1021#[derive(Debug, Clone, PartialEq, Eq)]
1022pub struct HealthConfig {
1023    /// Optional loopback HTTP endpoint for a managed non-wire process.
1024    /// Changing it applies live on rescan; the process protocol changes only
1025    /// at its next spawn.
1026    pub http: Option<String>,
1027    pub cadence: Duration,
1028    pub deadline: Duration,
1029    pub failure_threshold: u32,
1030    pub on_degraded: HealthAction,
1031    pub on_failing: HealthAction,
1032    pub critical: bool,
1033}
1034
1035impl Default for HealthConfig {
1036    fn default() -> Self {
1037        Self {
1038            http: None,
1039            cadence: DEFAULT_HEALTH_CADENCE,
1040            deadline: DEFAULT_HEALTH_DEADLINE,
1041            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1042            on_degraded: HealthAction::Report,
1043            on_failing: HealthAction::Report,
1044            critical: false,
1045        }
1046    }
1047}
1048
1049/// The supervisor's view of one module's health, relayed to clients over
1050/// channel-0 and rendered by `ck health`.
1051///
1052/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1053/// stated here rather than only at the wire type a consumer reads. A reader can
1054/// look up what `None` means; only a writer can silently change it, and the
1055/// writer has no reason to go looking at a downstream contract before editing.
1056///
1057/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1058/// back to `None` on re-registration precisely so a respawned module does not
1059/// carry its predecessor's timestamp — so an old value and an absent one call for
1060/// opposite readings, and anything that defaulted this to a number would make a
1061/// never-probed module indistinguishable from one probed at the epoch.
1062///
1063/// `detail` and `metrics` are `None` when the module published none on this
1064/// probe, which does not mean it reported nothing wrong — it is also the shape
1065/// when the probe never reached it. `last_probe_ms` is what separates those.
1066#[derive(Debug, Clone, PartialEq)]
1067pub struct ModuleHealthStatus {
1068    pub status: SupervisorHealthStatus,
1069    pub last_probe_ms: Option<u64>,
1070    pub detail: Option<String>,
1071    pub metrics: Option<Value>,
1072    pub consecutive_failures: u32,
1073    /// Number of replies received after a recurring health probe's deadline.
1074    /// Unlike a timeout, every increment proves the module was alive.
1075    pub late_answer_count: u64,
1076    /// End-to-end latency of the newest late reply, measured from probe start.
1077    pub last_late_answer_latency_ms: Option<u64>,
1078    pub last_action: Option<String>,
1079    /// Set together with `last_action`; the pair moves as one, and both being
1080    /// absent means no escalation has ever been taken rather than that the last
1081    /// one succeeded.
1082    pub last_action_ms: Option<u64>,
1083}
1084
1085impl Default for ModuleHealthStatus {
1086    fn default() -> Self {
1087        Self {
1088            status: SupervisorHealthStatus::Unknown,
1089            last_probe_ms: None,
1090            detail: None,
1091            metrics: None,
1092            consecutive_failures: 0,
1093            late_answer_count: 0,
1094            last_late_answer_latency_ms: None,
1095            last_action: None,
1096            last_action_ms: None,
1097        }
1098    }
1099}
1100
1101/// Typed lifecycle state for a supervised module.
1102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1103pub enum ModuleState {
1104    Starting,
1105    Running,
1106    Unresponsive,
1107    Restarting,
1108    Draining,
1109    Stopped,
1110    Failed,
1111    Disabled,
1112}
1113
1114impl fmt::Display for ModuleState {
1115    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1116        f.write_str(match self {
1117            Self::Starting => "starting",
1118            Self::Running => "running",
1119            Self::Unresponsive => "unresponsive",
1120            Self::Restarting => "restarting",
1121            Self::Draining => "draining",
1122            Self::Stopped => "stopped",
1123            Self::Failed => "failed",
1124            Self::Disabled => "disabled",
1125        })
1126    }
1127}
1128
1129/// Supervisor classification of a child-process exit.
1130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1131pub enum ExitKind {
1132    Clean,
1133    Crash,
1134    DeliberateSeverance,
1135}
1136
1137impl From<ExitKind> for TerminalExitKind {
1138    fn from(kind: ExitKind) -> Self {
1139        match kind {
1140            ExitKind::Clean => Self::Clean,
1141            ExitKind::Crash => Self::Crash,
1142            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1143        }
1144    }
1145}
1146
1147/// Exact process identity retained when a supervised module registers its
1148/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub(crate) struct ProcessIdentity {
1151    pub(crate) pid: u32,
1152    pub(crate) start_time: u64,
1153}
1154
1155/// Last observed child exit, if any.
1156#[derive(Debug, Clone, PartialEq, Eq)]
1157pub struct ExitReport {
1158    pub kind: ExitKind,
1159    pub code: Option<i32>,
1160    pub signal: Option<i32>,
1161    pub at_ms: u64,
1162}
1163
1164/// Point-in-time module status answerable by subc without forwarding to the
1165/// module process.
1166#[derive(Debug, Clone, PartialEq)]
1167pub struct ModuleStatus {
1168    pub module_id: String,
1169    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1170    pub waiting_for_previous_pid: Option<u32>,
1171    /// Why the previous instance still prevents a new start.
1172    pub waiting_for_previous_detail: Option<String>,
1173    /// The daemon's Windows per-spawn record: `pipe` after the process it
1174    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1175    /// List/status reports use this record, not the source the module sends
1176    /// in its registration message's manifest.provenance.launch_nonce_source.
1177    pub launch_nonce_source: Option<String>,
1178    pub state: ModuleState,
1179    pub enabled: bool,
1180    pub process_alive: bool,
1181    pub registration_active: bool,
1182    /// The module's declared wire protocol, carried beside `live` because it is
1183    /// what makes `live` readable: the two fields answer one question together.
1184    /// While a process is alive this is its launch declaration, not a later
1185    /// pending-reload edit. When down it is the configured next launch protocol.
1186    pub protocol: ModuleProtocol,
1187    /// Whether the module is serving, under the strongest definition the daemon
1188    /// can assert for its protocol.
1189    ///
1190    /// A subc module must also be REGISTERED: its process being alive says
1191    /// nothing about whether it can take a request. A `protocol: "none"` module
1192    /// never registers, so that term is dropped and this falls back to "enabled,
1193    /// running, and the process the daemon launched is alive" -- which is all
1194    /// the daemon observes about a process that speaks no subc wire. It stays a
1195    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1196    /// rather than printing it bare.
1197    pub live: bool,
1198    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1199    /// restarts have already released their slot, so this count can go down
1200    /// without anybody touching the module.
1201    pub restart_count: u32,
1202    /// Replacement processes spawned over this module's entire supervisor lifetime;
1203    /// unlike `restart_count`, this value is never reset by an operator action
1204    /// and never falls out of a window.
1205    pub lifetime_restarts: u32,
1206    pub spawn_generation: u64,
1207    /// The budget `restart_count` is spent against. Carried alongside the count
1208    /// because the count alone does not say how close the module is to being
1209    /// disabled, and reporting one without the other is what makes an
1210    /// about-to-be-retired module look ordinary.
1211    pub max_restarts: u32,
1212    /// The span `restart_count` is counted over. Carried with the pair above for
1213    /// the same reason they are carried together: "2 of 3" means one thing for a
1214    /// ten-minute window and something else entirely for a lifetime.
1215    pub restart_window: Duration,
1216    /// Effective drain and restart timing policy used by this running module.
1217    /// These values are carried together with the restart budget so status
1218    /// readers can compare configured intent with what the supervisor applied.
1219    pub drain_timeout: Duration,
1220    pub restart_backoff: Duration,
1221    pub restart_max_backoff: Duration,
1222    /// The module's process. On macOS this stays absent while the `ck-subc`
1223    /// launch trampoline is still running in that pid, and appears once the
1224    /// exec acknowledgement confirms the module image has replaced it. Launch
1225    /// time and the supervisor's own hold on the process are unaffected.
1226    pub pid: Option<u32>,
1227    pub spawned_at_ms: Option<u64>,
1228    pub spawned_from: Option<PathBuf>,
1229    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1230    pub process_start_time: Option<u64>,
1231    pub last_exit: Option<ExitReport>,
1232    pub health: ModuleHealthStatus,
1233}
1234
1235#[cfg(test)]
1236#[derive(Debug, Clone, Copy, PartialEq)]
1237struct ActorSelectCheckpoint {
1238    generation: u64,
1239    turn: u64,
1240    registered_connection: Option<ConnectionId>,
1241    next_probe_at: Option<Instant>,
1242    wake_after: Option<Duration>,
1243}
1244
1245#[derive(Debug, Clone, PartialEq)]
1246struct SupervisorSnapshot {
1247    failed_retry: FailedRetry,
1248    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1249    #[cfg(windows)]
1250    pipe_consumed: Option<Arc<OnceLock<()>>>,
1251    /// Counts running-child loop turns, not executor polls of a parked wait.
1252    #[cfg(test)]
1253    actor_turns: u64,
1254    /// Running select after exec confirmation and with no unconsumed registry
1255    /// event. Recording a spawn or starting a loop turn is not this barrier.
1256    #[cfg(test)]
1257    actor_select: Option<ActorSelectCheckpoint>,
1258    state: ModuleState,
1259    enabled: bool,
1260    process_alive: bool,
1261    spawned_protocol: Option<ModuleProtocol>,
1262    spawn_failure: Option<String>,
1263    waiting_for_previous_pid: Option<u32>,
1264    waiting_for_previous_detail: Option<String>,
1265    /// When each crash restart was spent, oldest first. This IS the crash
1266    /// budget: its in-window length is the count an operator sees and the count
1267    /// the restart decision is made against, so there is no second counter that
1268    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1269    /// operator actions that used to zero the old lifetime counter.
1270    crash_restarts: VecDeque<Instant>,
1271    lifetime_restarts: u32,
1272    /// Successful child spawns in this daemon incarnation.
1273    ///
1274    /// `lifetime_restarts` was considered and rejected: it starts at zero
1275    /// (line 640), successful initial/operator spawns in `set_running` do not
1276    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1277    /// increments before a successful replacement exists (lines 604, 3846,
1278    /// and 3921), so a failed spawn can consume it. This counter moves only
1279    /// when a live PID is accepted below.
1280    spawn_generation: u64,
1281    pid: Option<u32>,
1282    #[cfg(target_os = "macos")]
1283    report_ready: Option<Arc<OnceLock<()>>>,
1284    /// Last reaped child, retained after current process facts are cleared.
1285    reaped_pid: Option<u32>,
1286    /// Whether a replacement has already been scheduled for the current process.
1287    respawn_pending: bool,
1288    /// A second restart is waiting for the replacement already scheduled.
1289    coalesced_restart_pending: bool,
1290    spawned_at_ms: Option<u64>,
1291    spawned_from: Option<PathBuf>,
1292    launch_spec_digest: Option<LaunchSpecDigest>,
1293    spawned_file_identity: Option<SpawnedFileIdentity>,
1294    process_start_time: Option<u64>,
1295    deliberate_severance: Option<ProcessIdentity>,
1296    last_exit: Option<ExitReport>,
1297    /// Diagnostic attached to the next drain's terminal record, if any.
1298    drain_disposition_detail: Option<String>,
1299    health: ModuleHealthStatus,
1300    /// Whether the current process was started as a swap candidate and so
1301    /// lives in the module's alternate cgroup. The next swap's candidate takes
1302    /// the other one, so the two processes of a swap never share a cgroup. A
1303    /// plain spawn always uses the primary cgroup.
1304    in_alternate_slot: bool,
1305    /// Whether the current `Draining` state ends in a replacement process
1306    /// (restart, reload, health restart) rather than a stop. Only meaningful
1307    /// while `state` is `Draining`; every entry into that state rewrites it.
1308    /// It is what lets route.open answer the retryable `module_reloading` to a
1309    /// consumer that reaches a still-registered process mid-restart, instead of
1310    /// the `supervisor_not_live` a stop or disable deserves.
1311    draining_to_replace: bool,
1312    /// Whether a configuration update has been applied since the current
1313    /// process was spawned, so that process runs an older spec than the one
1314    /// the supervisor now holds. A queued restart is only coalesced into a
1315    /// fresher process when this is false: a restart requested to pick up a
1316    /// new configuration must not be satisfied by a process that predates it.
1317    configuration_updated_since_spawn: bool,
1318}
1319
1320impl SupervisorSnapshot {
1321    /// The pid that status, provenance and resource readings may report. While
1322    /// the launch trampoline still runs in the pid, reading its executable or
1323    /// resource use would describe `ck-subc`, not the module, so none is
1324    /// reported until the exec acknowledgement confirms the module image.
1325    fn reported_pid(&self) -> Option<u32> {
1326        #[cfg(target_os = "macos")]
1327        if self
1328            .report_ready
1329            .as_ref()
1330            .is_some_and(|ready| ready.get().is_none())
1331        {
1332            return None;
1333        }
1334        self.pid
1335    }
1336
1337    fn starting() -> Self {
1338        Self::new(ModuleState::Starting, true)
1339    }
1340
1341    fn disabled() -> Self {
1342        Self::new(ModuleState::Disabled, false)
1343    }
1344
1345    /// Crash restarts still inside `window`, having dropped the ones that are
1346    /// not. Pruning on read is what makes the budget a rate: an instant older
1347    /// than the window stops holding a slot the moment anybody counts.
1348    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1349        while let Some(oldest) = self.crash_restarts.front() {
1350            if now.duration_since(*oldest) > window {
1351                self.crash_restarts.pop_front();
1352            } else {
1353                break;
1354            }
1355        }
1356        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1357    }
1358
1359    /// Spend one unit of the crash budget and record the restart in the ledger.
1360    ///
1361    /// The ring is bounded by the cap because more than `max_restarts` in-window
1362    /// instants can never be reached (the caller refuses the restart first), so
1363    /// anything beyond that is an unbounded queue waiting to happen.
1364    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1365        self.crash_restarts.push_back(now);
1366        while self.crash_restarts.len() > policy.max_restarts as usize {
1367            self.crash_restarts.pop_front();
1368        }
1369        self.lifetime_restarts += 1;
1370    }
1371
1372    /// Reserve one crash-restart slot and calculate the delay before respawning.
1373    /// The count is captured before recording this restart, so the first retry
1374    /// uses the base delay and each later in-window retry escalates once.
1375    fn next_crash_restart(
1376        &mut self,
1377        policy: &RestartPolicy,
1378        now: Instant,
1379    ) -> Option<CrashRestartSchedule> {
1380        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1381        if restart_in_window >= policy.max_restarts {
1382            return None;
1383        }
1384        self.record_crash_restart(policy, now);
1385        Some(CrashRestartSchedule {
1386            restart_in_window,
1387            delay: policy.delay_for_restart(restart_in_window),
1388        })
1389    }
1390
1391    /// Give the module its full budget back, as an operator restart, reload, or
1392    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1393    /// ledger of what actually happened, and an operator action does not unmake
1394    /// the crashes.
1395    fn clear_crash_restarts(&mut self) {
1396        self.crash_restarts.clear();
1397    }
1398
1399    fn new(state: ModuleState, enabled: bool) -> Self {
1400        Self {
1401            failed_retry: FailedRetry::default(),
1402            exec_outcome: None,
1403            #[cfg(test)]
1404            actor_turns: 0,
1405            #[cfg(test)]
1406            actor_select: None,
1407            state,
1408            enabled,
1409            process_alive: false,
1410            #[cfg(windows)]
1411            pipe_consumed: None,
1412            spawned_protocol: None,
1413            spawn_failure: None,
1414            waiting_for_previous_pid: None,
1415            waiting_for_previous_detail: None,
1416            crash_restarts: VecDeque::new(),
1417            lifetime_restarts: 0,
1418            spawn_generation: 0,
1419            pid: None,
1420            #[cfg(target_os = "macos")]
1421            report_ready: None,
1422            reaped_pid: None,
1423            respawn_pending: false,
1424            coalesced_restart_pending: false,
1425            spawned_at_ms: None,
1426            spawned_from: None,
1427            launch_spec_digest: None,
1428            spawned_file_identity: None,
1429            process_start_time: None,
1430            deliberate_severance: None,
1431            last_exit: None,
1432            drain_disposition_detail: None,
1433            health: ModuleHealthStatus::default(),
1434            in_alternate_slot: false,
1435            draining_to_replace: false,
1436            configuration_updated_since_spawn: false,
1437        }
1438    }
1439}
1440
1441type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1442
1443type SpawnSubscriberKey = (ConnectionId, u64);
1444
1445#[derive(Debug)]
1446struct SpawnSubscriber {
1447    version: u8,
1448    frames: mpsc::Sender<Frame>,
1449    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1450    /// from which event. The full frame channel cannot carry that news, so it
1451    /// travels beside it; see `SpawnEventFeed::subscribe`.
1452    lagged: Option<oneshot::Sender<SpawnCursor>>,
1453}
1454
1455#[derive(Debug)]
1456struct SpawnEventState {
1457    daemon_incarnation: String,
1458    seq: u64,
1459    capacity: usize,
1460    live: HashMap<String, LiveSpawn>,
1461    generations: HashMap<String, u64>,
1462    events: VecDeque<SpawnEvent>,
1463    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1464}
1465
1466impl Default for SpawnEventState {
1467    fn default() -> Self {
1468        Self {
1469            daemon_incarnation: "unconfigured".to_string(),
1470            seq: 0,
1471            capacity: SPAWN_EVENT_RING_CAPACITY,
1472            live: HashMap::new(),
1473            generations: HashMap::new(),
1474            events: VecDeque::new(),
1475            subscribers: HashMap::new(),
1476        }
1477    }
1478}
1479
1480#[derive(Debug, Clone, Default)]
1481struct SpawnEventFeed(
1482    Arc<Mutex<SpawnEventState>>,
1483    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1484);
1485
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub(crate) enum SpawnSubscribeRefusal {
1488    ForeignIncarnation { current: String },
1489    TooOld { oldest: SpawnCursor },
1490    Frame(String),
1491}
1492
1493impl SpawnEventFeed {
1494    fn configure_incarnation(&self, daemon_incarnation: String) {
1495        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1496        state.daemon_incarnation = daemon_incarnation;
1497        state.seq = 0;
1498        state.live.clear();
1499        state.generations.clear();
1500        state.events.clear();
1501        state.subscribers.clear();
1502    }
1503
1504    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1505        SpawnCursor {
1506            daemon_incarnation: state.daemon_incarnation.clone(),
1507            seq: state.seq,
1508        }
1509    }
1510
1511    fn snapshot(&self) -> SpawnSnapshot {
1512        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1513        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1514        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1515        SpawnSnapshot {
1516            cursor: Self::cursor(&state),
1517            ring_bound: state.capacity as u64,
1518            live,
1519        }
1520    }
1521
1522    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1523        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1524        let generation = state
1525            .generations
1526            .get(module_id)
1527            .copied()
1528            .unwrap_or(0)
1529            .checked_add(1)
1530            .expect("spawn generation exhausted");
1531        state.generations.insert(module_id.to_string(), generation);
1532        let live = LiveSpawn {
1533            module_id: module_id.to_string(),
1534            spawn_generation: generation,
1535            pid,
1536            spawned_at_ms,
1537        };
1538        state.live.insert(module_id.to_string(), live);
1539        Self::emit_locked(
1540            &mut state,
1541            SpawnEventKind::Spawned,
1542            module_id.to_string(),
1543            generation,
1544            pid,
1545            None,
1546            None,
1547        );
1548        generation
1549    }
1550
1551    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1552        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1553        let Some(live) = state.live.remove(module_id) else {
1554            warn!(
1555                module_id,
1556                "terminal record had no live spawn event identity"
1557            );
1558            return;
1559        };
1560        Self::emit_locked(
1561            &mut state,
1562            SpawnEventKind::Exited,
1563            module_id.to_string(),
1564            live.spawn_generation,
1565            live.pid,
1566            exit_code,
1567            exit_signal,
1568        );
1569    }
1570
1571    /// Report the exit of a process that a swap has already replaced.
1572    ///
1573    /// `emit_exited` removes the module's live entry, which after a swap's
1574    /// cutover describes the promoted candidate, not the old process now
1575    /// exiting. This emits the old generation's exit and leaves the live entry
1576    /// alone unless it still names that generation.
1577    fn emit_superseded_exited(
1578        &self,
1579        module_id: &str,
1580        spawn_generation: u64,
1581        pid: u32,
1582        exit_code: Option<i32>,
1583        exit_signal: Option<i32>,
1584    ) {
1585        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1586        if state
1587            .live
1588            .get(module_id)
1589            .is_some_and(|live| live.spawn_generation == spawn_generation)
1590        {
1591            state.live.remove(module_id);
1592        }
1593        Self::emit_locked(
1594            &mut state,
1595            SpawnEventKind::Exited,
1596            module_id.to_string(),
1597            spawn_generation,
1598            pid,
1599            exit_code,
1600            exit_signal,
1601        );
1602    }
1603
1604    #[allow(clippy::too_many_arguments)]
1605    fn emit_locked(
1606        state: &mut SpawnEventState,
1607        kind: SpawnEventKind,
1608        module_id: String,
1609        spawn_generation: u64,
1610        pid: u32,
1611        exit_code: Option<i32>,
1612        exit_signal: Option<i32>,
1613    ) {
1614        state.seq = state
1615            .seq
1616            .checked_add(1)
1617            .expect("spawn event sequence exhausted");
1618        let event = SpawnEvent {
1619            cursor: Self::cursor(state),
1620            kind,
1621            module_id,
1622            spawn_generation,
1623            pid,
1624            exit_code,
1625            exit_signal,
1626        };
1627        state.events.push_back(event.clone());
1628        while state.events.len() > state.capacity {
1629            state.events.pop_front();
1630        }
1631        let body = match serde_json::to_vec(&event) {
1632            Ok(body) => body,
1633            Err(error) => {
1634                error!(%error, "failed to serialize supervisor spawn event");
1635                return;
1636            }
1637        };
1638        state.subscribers.retain(|(connection_id, corr), subscriber| {
1639            let frame = Frame::build_with_version(
1640                subscriber.version,
1641                FrameType::StreamData,
1642                control_flags(),
1643                0,
1644                0,
1645                *corr,
1646                body.clone(),
1647            );
1648            match frame {
1649                Ok(frame) => {
1650                    if subscriber.frames.try_send(frame).is_ok() {
1651                        true
1652                    } else {
1653                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1654                        if let Some(lagged) = subscriber.lagged.take() {
1655                            let _ = lagged.send(event.cursor.clone());
1656                        }
1657                        false
1658                    }
1659                }
1660                Err(error) => {
1661                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1662                    false
1663                }
1664            }
1665        });
1666    }
1667
1668    fn subscribe(
1669        &self,
1670        connection_id: ConnectionId,
1671        corr: u64,
1672        version: u8,
1673        since: Option<SpawnCursor>,
1674        sink: FrameSink,
1675    ) -> Result<(), SpawnSubscribeRefusal> {
1676        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1677        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1678        {
1679            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1680            let replay = if let Some(since) = since {
1681                if since.daemon_incarnation != state.daemon_incarnation {
1682                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1683                        current: state.daemon_incarnation.clone(),
1684                    });
1685                }
1686                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1687                    if since.seq < oldest.seq.saturating_sub(1) {
1688                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1689                    }
1690                }
1691                state
1692                    .events
1693                    .iter()
1694                    .filter(|event| event.cursor.seq > since.seq)
1695                    .cloned()
1696                    .collect::<Vec<_>>()
1697            } else {
1698                Vec::new()
1699            };
1700            for event in replay {
1701                let body = serde_json::to_vec(&event)
1702                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1703                let frame = Frame::build_with_version(
1704                    version,
1705                    FrameType::StreamData,
1706                    control_flags(),
1707                    0,
1708                    0,
1709                    corr,
1710                    body,
1711                )
1712                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1713                frames
1714                    .try_send(frame)
1715                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716            }
1717            state.subscribers.insert(
1718                (connection_id, corr),
1719                SpawnSubscriber {
1720                    version,
1721                    frames,
1722                    lagged: Some(lagged),
1723                },
1724            );
1725        }
1726        // The lagged terminal is sent here, by the forwarder, rather than by
1727        // the emitter: at the moment of the drop the subscriber's own channel
1728        // is full, and writing to the connection sink directly from the emitter
1729        // would put the Error AHEAD of the events still queued in that channel
1730        // (and the emitter holds the feed lock, so it cannot await the sink).
1731        // Dropping the subscriber drops the only sender, so `recv` drains every
1732        // queued event and then returns `None`; only then is the Error sent, so
1733        // the client sees each event it can keep, then the reason it was cut.
1734        // Cancel and connection removal drop the oneshot unsent, so they end
1735        // the stream with no Error.
1736        tokio::spawn(async move {
1737            while let Some(frame) = receiver.recv().await {
1738                if sink.send(frame).await.is_err() {
1739                    return;
1740                }
1741            }
1742            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1743                return;
1744            };
1745            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1746                Ok(frame) => {
1747                    let _ = sink.send(frame).await;
1748                }
1749                Err(error) => {
1750                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1751                }
1752            }
1753        });
1754        Ok(())
1755    }
1756
1757    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1758        let Some(subscriber) = self
1759            .0
1760            .lock()
1761            .unwrap_or_else(|p| p.into_inner())
1762            .subscribers
1763            .remove(&(connection_id, corr))
1764        else {
1765            return false;
1766        };
1767        if let Ok(frame) = Frame::build_with_version(
1768            subscriber.version,
1769            FrameType::StreamEnd,
1770            control_flags(),
1771            0,
1772            0,
1773            corr,
1774            Vec::new(),
1775        ) {
1776            tokio::spawn(async move {
1777                let _ = subscriber.frames.send(frame).await;
1778            });
1779        }
1780        true
1781    }
1782
1783    fn remove_connection(&self, connection_id: ConnectionId) {
1784        self.0
1785            .lock()
1786            .unwrap_or_else(|p| p.into_inner())
1787            .subscribers
1788            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1789    }
1790
1791    #[cfg(any(test, feature = "test-support"))]
1792    fn set_capacity(&self, capacity: usize) {
1793        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1794    }
1795
1796    #[cfg(any(test, feature = "test-support"))]
1797    fn subscriber_count(&self) -> usize {
1798        self.0
1799            .lock()
1800            .unwrap_or_else(|p| p.into_inner())
1801            .subscribers
1802            .len()
1803    }
1804}
1805
1806/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1807/// The terminal Error a lagged spawn subscriber receives after its queued events.
1808fn spawn_subscriber_lagged_frame(
1809    version: u8,
1810    corr: u64,
1811    first_undelivered: SpawnCursor,
1812) -> Result<Frame, String> {
1813    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1814    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1815        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1816        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1817            .to_string(),
1818        detail: Some(serde_json::json!({
1819            "first_undelivered_cursor": first_undelivered
1820        })),
1821    })
1822    .map_err(|error| error.to_string())?;
1823    Frame::build_with_version(
1824        version,
1825        FrameType::Error,
1826        crate::router::daemon_error_flags(),
1827        0,
1828        0,
1829        corr,
1830        body,
1831    )
1832    .map_err(|error| error.to_string())
1833}
1834
1835pub trait ModuleProcessLiveness: Send + Sync {
1836    fn process_live(&self, module_id: &str) -> Option<bool>;
1837
1838    /// Whether the supervisor is replacing this module's process right now: an
1839    /// operator restart or reload, a health restart, or a crash respawn whose
1840    /// backoff is running. A module in that state is not live, but a consumer
1841    /// refused now should retry shortly rather than treat the target as gone.
1842    /// Stopped, failed, and disabled modules are not replacing.
1843    fn process_replacing(&self, _module_id: &str) -> bool {
1844        false
1845    }
1846}
1847
1848/// Shared process-liveness registry keyed by supervised `module_id`.
1849#[derive(Debug, Clone, Default)]
1850pub struct SupervisorProcessLiveness {
1851    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1852}
1853
1854impl SupervisorProcessLiveness {
1855    pub fn new() -> Self {
1856        Self::default()
1857    }
1858
1859    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1860        let mut snapshots = self
1861            .snapshots
1862            .lock()
1863            .unwrap_or_else(|poisoned| poisoned.into_inner());
1864        snapshots.insert(module_id, snapshot);
1865    }
1866
1867    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1868        let mut snapshots = self
1869            .snapshots
1870            .lock()
1871            .unwrap_or_else(|poisoned| poisoned.into_inner());
1872        let is_current = snapshots
1873            .get(module_id)
1874            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1875            .unwrap_or(false);
1876        if is_current {
1877            snapshots.remove(module_id);
1878        }
1879    }
1880}
1881
1882impl ModuleProcessLiveness for SupervisorProcessLiveness {
1883    fn process_live(&self, module_id: &str) -> Option<bool> {
1884        let snapshot = {
1885            let snapshots = self
1886                .snapshots
1887                .lock()
1888                .unwrap_or_else(|poisoned| poisoned.into_inner());
1889            snapshots.get(module_id).cloned()
1890        }?;
1891        let snapshot = snapshot
1892            .lock()
1893            .unwrap_or_else(|poisoned| poisoned.into_inner());
1894        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1895    }
1896
1897    fn process_replacing(&self, module_id: &str) -> bool {
1898        let Some(snapshot) = self
1899            .snapshots
1900            .lock()
1901            .unwrap_or_else(|poisoned| poisoned.into_inner())
1902            .get(module_id)
1903            .cloned()
1904        else {
1905            return false;
1906        };
1907        let snapshot = snapshot
1908            .lock()
1909            .unwrap_or_else(|poisoned| poisoned.into_inner());
1910        snapshot.enabled
1911            && match snapshot.state {
1912                ModuleState::Restarting => true,
1913                ModuleState::Draining => snapshot.draining_to_replace,
1914                ModuleState::Starting
1915                | ModuleState::Running
1916                | ModuleState::Unresponsive
1917                | ModuleState::Stopped
1918                | ModuleState::Failed
1919                | ModuleState::Disabled => false,
1920            }
1921    }
1922}
1923
1924#[cfg(test)]
1925#[derive(Debug, Default)]
1926struct ReloadExitRecordGate {
1927    reached: tokio::sync::Notify,
1928    resume: tokio::sync::Notify,
1929}
1930
1931#[derive(Debug, Clone, Copy)]
1932enum RespawnKind {
1933    Spawn,
1934    Reload,
1935}
1936
1937#[derive(Debug, Clone, Copy)]
1938struct PendingRespawn {
1939    deadline: Instant,
1940    kind: RespawnKind,
1941    daemon_initiated: bool,
1942}
1943
1944type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1945
1946#[derive(Debug, Clone)]
1947struct SupervisorRuntimeConfig {
1948    registry: Arc<Registry>,
1949    /// Retained across an operation's deferred backoff, until its child starts.
1950    recovery_is_daemon: bool,
1951    start_requester: Option<Principal>,
1952    pending_activity_restart: bool,
1953    provenance_probe: ExecutableIdentityProbe,
1954    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1955    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1956    /// A reload acknowledges completion only after its replacement registers.
1957    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1958    restart_policy: RestartPolicy,
1959    register_timeout: Duration,
1960    /// This module's RESOLVED drain budget: per-module config when present,
1961    /// else `default_drain_timeout`.
1962    drain_timeout: Duration,
1963    /// Shared with the status handle so the attested value changes atomically
1964    /// when a rescan updates the running drain policy.
1965    effective_drain_timeout: Arc<Mutex<Duration>>,
1966    /// The supervisor-wide fallback, kept so a configuration update that
1967    /// REMOVES the per-module override can re-resolve to it.
1968    default_drain_timeout: Duration,
1969    health: HealthConfig,
1970    connection_file_path: Option<PathBuf>,
1971    capture_logs_dir: Option<PathBuf>,
1972    forwarding: Option<Arc<ForwardingTable>>,
1973    /// The shared handle, so every spawn path (initial, restart, reload) records the
1974    /// reserved-module launch nonce the HELLO verifier checks against.
1975    supervisor_handle: Option<SupervisorHandle>,
1976    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1977    /// status queries.
1978    ///
1979    /// One ring per module, held across every respawn. The lines explaining an exit
1980    /// are written BEFORE that exit, so a ring recreated per process would be empty
1981    /// exactly when it is asked for.
1982    stderr_ring: Arc<Mutex<StderrRing>>,
1983    terminal_ring: Arc<Mutex<TerminalRing>>,
1984    spawn_events: SpawnEventFeed,
1985    child_roster: ChildRoster,
1986    #[cfg(target_os = "linux")]
1987    cgroup_placement: Option<subc_cgroup::Placement>,
1988    #[cfg(test)]
1989    test_seed_stale_facts_before_enable_spawn: bool,
1990    #[cfg(test)]
1991    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1992}
1993
1994impl SupervisorRuntimeConfig {
1995    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1996        FailureContext {
1997            daemon_initiated,
1998            shutdown: self.child_roster.shutdown_flag(),
1999        }
2000    }
2001}
2002
2003#[derive(Debug, Clone, PartialEq, Eq)]
2004struct SupervisedConfiguration {
2005    spec: ModuleSpec,
2006    health: HealthConfig,
2007}
2008
2009/// Shared daemon lookup table for supervised module handles.
2010///
2011/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2012/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2013/// launch nonces recorded at spawn are checked by the same daemon instance.
2014#[derive(Debug, Clone, Default)]
2015pub struct SupervisorHandle {
2016    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2017    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2018    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2019    /// Module ids the supervisor has taken on. An id is added BEFORE the
2020    /// module's first process is spawned and removed only when the module
2021    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2022    /// the keys of `modules`.
2023    ///
2024    /// `modules` cannot answer "is this module configured?" on its own: a
2025    /// [`SupervisedModule`] only exists once its process has been spawned, and
2026    /// a fast child can connect, register, sync its scopes and ask about them
2027    /// before the supervisor has inserted it. Answering "not configured" in that
2028    /// gap makes scope admission refuse with the terminal "will never sync"
2029    /// instead of the retryable "has not synced yet".
2030    configured_ids: Arc<Mutex<HashSet<String>>>,
2031    spawn_events: SpawnEventFeed,
2032    /// The current expected launch nonce for each reserved module_id. Set when the
2033    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2034    /// non-reserved module never has an entry here and is never nonce-checked.
2035    /// Reserved module ids and the nonce that authorizes their next HELLO.
2036    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2037    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2038    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2039    /// had NO entry and admitted anyone: the reservation protected the nonce
2040    /// holder, not the NAME (found live by CKCRED's canary probe registering
2041    /// against a reserved scratch id).
2042    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2043    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2044    ///
2045    /// This is deliberately in-memory only: subc is state-free across daemon
2046    /// restarts, and the tombstone only explains the hours-after-removal window
2047    /// while this executing daemon is still alive. Do not persist it in a store.
2048    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2049    /// The current launch nonce for every supervised spawn. This is separate from
2050    /// reserved_nonces because consumer route.open attestation applies to all spawned
2051    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2052    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2053    /// Reserved namespace prefixes mapped to the supervised owner module whose
2054    /// current spawn nonce authorizes HELLO claims below the prefix.
2055    ///
2056    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2057    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2058    /// accidental collisions and lower-trust processes from squatting protected
2059    /// namespaces.
2060    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2061    /// Blue/green swaps in progress, by module id. An entry exists from just
2062    /// before the candidate process is spawned until the swap has failed, or
2063    /// has cut over and the old process is gone. While it exists, HELLO for the
2064    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2065    /// consumer attestation accepts both processes' nonces.
2066    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2067    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2068    promotion_observer: PromotionObserverSlot,
2069    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2070    /// this daemon-wide ordering, a rescan could retire or update a module while a
2071    /// concurrent reload still held its old handle and launch specification.
2072    operation_lock: Arc<AsyncMutex<()>>,
2073}
2074
2075/// Told when a swap has promoted its candidate to be the module's active
2076/// registration.
2077///
2078/// An ordinary HELLO runs the control plane's registration side effects (the
2079/// capability cache, the deny census, the requirement recompute) as it
2080/// registers. A swap candidate's HELLO does not, because it is not routable;
2081/// promotion is when those must run instead, and promotion happens in the
2082/// supervisor, which has no other way into the control handler.
2083pub(crate) trait SwapPromotionObserver: Send + Sync {
2084    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2085}
2086
2087/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2088/// control handler) owns this handle, so a strong reference back would be a
2089/// cycle that keeps both alive.
2090#[derive(Clone, Default)]
2091struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2092
2093impl fmt::Debug for PromotionObserverSlot {
2094    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2095        f.write_str("PromotionObserverSlot")
2096    }
2097}
2098
2099/// The nonces of one open swap.
2100#[derive(Debug, Clone)]
2101struct OpenSwap {
2102    /// The launch nonce minted for the candidate process. It is the swap
2103    /// token: the only thing that admits a HELLO into the candidate slot.
2104    candidate_nonce: String,
2105    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2106    /// here because cutover moves the module's recorded spawn nonce to the
2107    /// candidate while the incumbent is still draining and its consumers are
2108    /// still attesting with this one.
2109    incumbent_nonce: Option<String>,
2110    /// Set once a HELLO has been admitted with the swap token, so the token
2111    /// admits one registration and cannot be replayed after cutover empties
2112    /// the candidate slot.
2113    candidate_admitted: bool,
2114}
2115
2116/// What the swap gate says about a HELLO. See
2117/// [`SupervisorHandle::swap_hello_admission`].
2118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2119pub(crate) enum SwapHelloAdmission {
2120    /// No swap is open for the id (or the HELLO carries the incumbent's own
2121    /// nonce); the ordinary gates decide.
2122    NotSwapping,
2123    /// The HELLO carries the swap token: register it into the candidate slot.
2124    Candidate,
2125    /// A swap is open and the HELLO carries a nonce the supervisor did not
2126    /// mint for this id, no nonce, or a token already used.
2127    Refused,
2128}
2129
2130#[derive(Debug, Clone, PartialEq, Eq)]
2131pub(crate) enum ReservedHelloRejection {
2132    Exact {
2133        module_id: String,
2134    },
2135    Prefix {
2136        prefix: String,
2137        owner_module_id: String,
2138    },
2139}
2140
2141impl SupervisorHandle {
2142    pub fn new() -> Self {
2143        Self::default()
2144    }
2145
2146    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2147        self.spawn_events.snapshot()
2148    }
2149
2150    pub(crate) fn subscribe_spawns(
2151        &self,
2152        connection_id: ConnectionId,
2153        corr: u64,
2154        version: u8,
2155        since: Option<SpawnCursor>,
2156        sink: FrameSink,
2157    ) -> Result<(), SpawnSubscribeRefusal> {
2158        self.spawn_events
2159            .subscribe(connection_id, corr, version, since, sink)
2160    }
2161
2162    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2163        self.spawn_events.cancel(connection_id, corr)
2164    }
2165
2166    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2167        self.spawn_events.remove_connection(connection_id);
2168    }
2169
2170    #[cfg(any(test, feature = "test-support"))]
2171    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2172        assert!(capacity > 0, "spawn event capacity must be non-zero");
2173        self.spawn_events.set_capacity(capacity);
2174    }
2175
2176    #[cfg(any(test, feature = "test-support"))]
2177    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2178        self.spawn_events.subscriber_count()
2179    }
2180
2181    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2182    /// a respawn invalidates stale consumer identities.
2183    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2184        self.spawn_nonces
2185            .lock()
2186            .unwrap_or_else(|poisoned| poisoned.into_inner())
2187            .insert(module_id.to_string(), nonce);
2188    }
2189
2190    /// Record the launch nonce expected from the next HELLO for a reserved module,
2191    /// replacing any prior nonce (a respawn invalidates the previous one).
2192    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2193        self.reserved_nonces
2194            .lock()
2195            .unwrap_or_else(|poisoned| poisoned.into_inner())
2196            .insert(module_id.to_string(), Some(nonce));
2197    }
2198
2199    /// Record namespace prefixes owned by a supervised module.
2200    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2201        let mut owners = self
2202            .reserved_prefix_owners
2203            .lock()
2204            .unwrap_or_else(|poisoned| poisoned.into_inner());
2205        owners.retain(|_, owner| owner != owner_module_id);
2206        for prefix in prefixes {
2207            owners.insert(prefix.clone(), owner_module_id.to_string());
2208        }
2209    }
2210
2211    /// The launch nonce most recently minted for a module's spawn, if any.
2212    #[cfg(test)]
2213    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2214        self.spawn_nonces
2215            .lock()
2216            .unwrap_or_else(|poisoned| poisoned.into_inner())
2217            .get(module_id)
2218            .cloned()
2219    }
2220
2221    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2222        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2223        let spawn_nonce = self
2224            .spawn_nonces
2225            .lock()
2226            .unwrap_or_else(|poisoned| poisoned.into_inner())
2227            .get(&spec.module_id)
2228            .cloned();
2229        let mut reserved_nonces = self
2230            .reserved_nonces
2231            .lock()
2232            .unwrap_or_else(|poisoned| poisoned.into_inner());
2233        if spec.reserved {
2234            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2235            // reserved name whose module has never spawned has no legitimate
2236            // holder, and the entry's absence is what used to leave the name
2237            // open to the first claimant.
2238            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2239        }
2240        drop(reserved_nonces);
2241        // A later unreserved declaration must not silently unreserve an id that
2242        // was retained after its reserved configuration was removed. The explicit
2243        // release ceremony is the only operation that retires that gate.
2244        self.removal_tombstones
2245            .lock()
2246            .unwrap_or_else(|poisoned| poisoned.into_inner())
2247            .remove(&spec.module_id);
2248    }
2249
2250    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2251    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2252    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2253    /// with no matching prefix are always authorized.
2254    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2255        self.reserved_hello_verification(module_id, presented)
2256            .is_ok()
2257    }
2258
2259    /// Whether HELLO passed a nonce check that was actually required by reserved
2260    /// module policy, or the refusal explaining why a required check failed.
2261    pub(crate) fn reserved_hello_verification(
2262        &self,
2263        module_id: &str,
2264        presented: Option<&str>,
2265    ) -> Result<bool, ReservedHelloRejection> {
2266        let nonces = self
2267            .reserved_nonces
2268            .lock()
2269            .unwrap_or_else(|poisoned| poisoned.into_inner());
2270        if let Some(expected) = nonces.get(module_id) {
2271            // `None` = reserved with no legitimate holder: refuse every
2272            // presentation, because no process can hold a nonce that was never
2273            // minted. Only a real minted nonce admits, in constant time.
2274            let authorized = match expected {
2275                Some(expected) => {
2276                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2277                }
2278                None => false,
2279            };
2280            if authorized {
2281                return Ok(true);
2282            }
2283            return Err(ReservedHelloRejection::Exact {
2284                module_id: module_id.to_string(),
2285            });
2286        }
2287        drop(nonces);
2288
2289        let matched_prefix = self
2290            .reserved_prefix_owners
2291            .lock()
2292            .unwrap_or_else(|poisoned| poisoned.into_inner())
2293            .iter()
2294            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2295            .max_by_key(|(prefix, _)| prefix.len())
2296            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2297        let Some((prefix, owner_module_id)) = matched_prefix else {
2298            return Ok(false);
2299        };
2300
2301        let authorized = presented.is_some_and(|presented| {
2302            self.spawn_nonces
2303                .lock()
2304                .unwrap_or_else(|poisoned| poisoned.into_inner())
2305                .get(&owner_module_id)
2306                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2307                // While the owner is being swapped, children started by
2308                // either of its two processes hold that process's nonce.
2309                || self.swap_nonce_matches(&owner_module_id, presented)
2310        });
2311        if authorized {
2312            Ok(true)
2313        } else {
2314            Err(ReservedHelloRejection::Prefix {
2315                prefix,
2316                owner_module_id,
2317            })
2318        }
2319    }
2320
2321    /// Whether a consumer connection proved it came from a daemon-spawned module.
2322    ///
2323    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2324    /// accepted only for module ids the supervisor has spawned.
2325    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2326        if presented.is_empty() {
2327            return false;
2328        }
2329        let nonces = self
2330            .spawn_nonces
2331            .lock()
2332            .unwrap_or_else(|poisoned| poisoned.into_inner());
2333        let current = nonces
2334            .get(module_id)
2335            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2336        drop(nonces);
2337        // During a swap two processes of the module are alive, and a consumer
2338        // started by either one presents that process's nonce. Accepting only
2339        // the recorded one would fail the incumbent's consumers for the whole
2340        // overlap once cutover moves the record to the candidate.
2341        current || self.swap_nonce_matches(module_id, presented)
2342    }
2343
2344    /// Whether `presented` is either nonce of an open swap for `module_id`.
2345    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2346        let swaps = self
2347            .swaps
2348            .lock()
2349            .unwrap_or_else(|poisoned| poisoned.into_inner());
2350        swaps.get(module_id).is_some_and(|swap| {
2351            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2352                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2353                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2354                })
2355        })
2356    }
2357
2358    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2359    /// Called before the candidate process exists.
2360    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2361        let incumbent_nonce = self
2362            .spawn_nonces
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner())
2365            .get(module_id)
2366            .cloned();
2367        self.swaps
2368            .lock()
2369            .unwrap_or_else(|poisoned| poisoned.into_inner())
2370            .insert(
2371                module_id.to_string(),
2372                OpenSwap {
2373                    candidate_nonce,
2374                    incumbent_nonce,
2375                    candidate_admitted: false,
2376                },
2377            );
2378    }
2379
2380    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2381    /// the module's recorded one.
2382    pub(crate) fn close_swap(&self, module_id: &str) {
2383        self.swaps
2384            .lock()
2385            .unwrap_or_else(|poisoned| poisoned.into_inner())
2386            .remove(module_id);
2387    }
2388
2389    /// Install the observer told about swap promotions, replacing any earlier
2390    /// one.
2391    pub(crate) fn set_swap_promotion_observer(
2392        &self,
2393        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2394    ) {
2395        *self
2396            .promotion_observer
2397            .0
2398            .lock()
2399            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2400    }
2401
2402    /// Tell the installed observer, if it is still alive, that a swap promoted
2403    /// `registration`.
2404    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2405        let observer = self
2406            .promotion_observer
2407            .0
2408            .lock()
2409            .unwrap_or_else(|poisoned| poisoned.into_inner())
2410            .as_ref()
2411            .and_then(std::sync::Weak::upgrade);
2412        if let Some(observer) = observer {
2413            observer.swap_promoted(registration);
2414        }
2415    }
2416
2417    /// Whether a swap is open for `module_id`.
2418    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2419        self.swaps
2420            .lock()
2421            .unwrap_or_else(|poisoned| poisoned.into_inner())
2422            .contains_key(module_id)
2423    }
2424
2425    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2426    /// respawn would, once cutover has made the candidate the module's process.
2427    /// The swap stays open so the incumbent's nonce keeps attesting until the
2428    /// incumbent has drained and exited.
2429    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2430        let candidate_nonce = self
2431            .swaps
2432            .lock()
2433            .unwrap_or_else(|poisoned| poisoned.into_inner())
2434            .get(module_id)
2435            .map(|swap| swap.candidate_nonce.clone());
2436        let Some(nonce) = candidate_nonce else {
2437            return;
2438        };
2439        self.set_spawn_nonce(module_id, nonce.clone());
2440        if reserved {
2441            self.set_reserved_nonce(module_id, nonce);
2442        }
2443    }
2444
2445    /// The swap gate for a HELLO claiming `module_id`.
2446    ///
2447    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2448    /// presents the candidate nonce, which the reserved gate (holding the
2449    /// incumbent's nonce) would refuse as `reserved_module` before swap
2450    /// admission was ever reached. And it applies to unreserved ids too: for an
2451    /// unreserved id the only thing that ever stopped a second process claiming
2452    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2453    /// refusal a swap lifts for its candidate.
2454    ///
2455    /// The incumbent's own nonce falls through to the ordinary gates, which
2456    /// treat it as they always have (a live incumbent is refused as a
2457    /// duplicate). Anything else while a swap is open is refused, including an
2458    /// absent nonce.
2459    pub(crate) fn swap_hello_admission(
2460        &self,
2461        module_id: &str,
2462        presented: Option<&str>,
2463    ) -> SwapHelloAdmission {
2464        let swaps = self
2465            .swaps
2466            .lock()
2467            .unwrap_or_else(|poisoned| poisoned.into_inner());
2468        let Some(swap) = swaps.get(module_id) else {
2469            return SwapHelloAdmission::NotSwapping;
2470        };
2471        let Some(presented) = presented else {
2472            return SwapHelloAdmission::Refused;
2473        };
2474        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2475            return if swap.candidate_admitted {
2476                SwapHelloAdmission::Refused
2477            } else {
2478                SwapHelloAdmission::Candidate
2479            };
2480        }
2481        if swap
2482            .incumbent_nonce
2483            .as_deref()
2484            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2485        {
2486            return SwapHelloAdmission::NotSwapping;
2487        }
2488        SwapHelloAdmission::Refused
2489    }
2490
2491    /// Record that the swap token has registered a candidate, so it admits no
2492    /// second HELLO.
2493    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2494        if let Some(swap) = self
2495            .swaps
2496            .lock()
2497            .unwrap_or_else(|poisoned| poisoned.into_inner())
2498            .get_mut(module_id)
2499        {
2500            swap.candidate_admitted = true;
2501        }
2502    }
2503
2504    /// Test/support lookup for the current launch nonce of a supervised spawn.
2505    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2506        self.spawn_nonces
2507            .lock()
2508            .unwrap_or_else(|poisoned| poisoned.into_inner())
2509            .get(module_id)
2510            .cloned()
2511    }
2512
2513    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2514    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2515        self.reserved_nonces
2516            .lock()
2517            .unwrap_or_else(|poisoned| poisoned.into_inner())
2518            .get(module_id)
2519            .cloned()
2520            .flatten()
2521    }
2522
2523    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2524        // Normally already marked before the process was spawned; marking here
2525        // too keeps `configured_ids` a superset of the roster for any caller
2526        // that inserts a module directly.
2527        self.mark_configured(module.module_id());
2528        let mut modules = self
2529            .modules
2530            .lock()
2531            .unwrap_or_else(|poisoned| poisoned.into_inner());
2532        modules.insert(module.module_id().to_string(), module)
2533    }
2534
2535    /// Record that the supervisor has taken on `module_id`. Called before the
2536    /// module's first process is spawned, so that by the time that process can
2537    /// register, [`Self::is_configured`] already answers true.
2538    fn mark_configured(&self, module_id: &str) {
2539        self.configured_ids
2540            .lock()
2541            .unwrap_or_else(|poisoned| poisoned.into_inner())
2542            .insert(module_id.to_string());
2543    }
2544
2545    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2546    /// before it was ever put on the roster. A module already on the roster
2547    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2548    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2549        let modules = self
2550            .modules
2551            .lock()
2552            .unwrap_or_else(|poisoned| poisoned.into_inner());
2553        if !modules.contains_key(module_id) {
2554            self.configured_ids
2555                .lock()
2556                .unwrap_or_else(|poisoned| poisoned.into_inner())
2557                .remove(module_id);
2558        }
2559    }
2560
2561    /// Whether `module_id` is a module this daemon supervises: on the roster,
2562    /// or about to be (its process is being spawned right now).
2563    ///
2564    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2565    /// for scopes: a supervised module's process can register and sync before
2566    /// [`Self::get`] can return it, and in that window it is still a module
2567    /// that will sync, not one that never will.
2568    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2569        self.configured_ids
2570            .lock()
2571            .unwrap_or_else(|poisoned| poisoned.into_inner())
2572            .contains(module_id)
2573    }
2574
2575    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2576        let modules = self
2577            .modules
2578            .lock()
2579            .unwrap_or_else(|poisoned| poisoned.into_inner());
2580        modules.get(module_id).cloned()
2581    }
2582
2583    pub(crate) fn record_late_health_answer(
2584        &self,
2585        module_id: &str,
2586        latency_ms: u64,
2587    ) -> Result<bool, SuperviseError> {
2588        let Some(module) = self.get(module_id) else {
2589            return Ok(false);
2590        };
2591        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2592            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2593            state.health.last_late_answer_latency_ms = Some(latency_ms);
2594            // A late answer is an answer: the module served the probe, just past
2595            // the deadline. Leaving the miss streak in place while logging
2596            // "proves the module is alive" is how a CPU-starved module that
2597            // answers every probe a few seconds late still marches to the
2598            // threshold and gets killed — the exact kill class `NoAnswer` is
2599            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2600            // is degradation, and degradation reports; it does not restart.
2601            state.health.consecutive_failures = 0;
2602        })?;
2603        Ok(true)
2604    }
2605
2606    /// Arm the one-shot marker for the module process that this caller
2607    /// deliberately initiated severance against. Generic connection teardown
2608    /// must not call this:
2609    /// a surviving process would otherwise retain an exemption for a later
2610    /// genuine crash.
2611    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2612        let Some(module) = self.get(module_id) else {
2613            return Ok(false);
2614        };
2615        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2616        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2617            return Ok(false);
2618        };
2619        drop(snapshot);
2620        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2621    }
2622
2623    pub fn list(&self) -> Vec<SupervisedModule> {
2624        let modules = self
2625            .modules
2626            .lock()
2627            .unwrap_or_else(|poisoned| poisoned.into_inner());
2628        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2629        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2630        modules
2631    }
2632
2633    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2634        self.activity_images
2635            .lock()
2636            .unwrap_or_else(|p| p.into_inner())
2637            .remove(module_id);
2638        self.spawn_nonces
2639            .lock()
2640            .unwrap_or_else(|poisoned| poisoned.into_inner())
2641            .remove(module_id);
2642        self.close_swap(module_id);
2643        let mut reserved_nonces = self
2644            .reserved_nonces
2645            .lock()
2646            .unwrap_or_else(|poisoned| poisoned.into_inner());
2647        if reserved_nonces.contains_key(module_id) {
2648            // The old nonce must die with the removed process, but the exact-id
2649            // gate remains until an operator explicitly releases it.
2650            reserved_nonces.insert(module_id.to_string(), None);
2651        }
2652        drop(reserved_nonces);
2653        self.reserved_prefix_owners
2654            .lock()
2655            .unwrap_or_else(|poisoned| poisoned.into_inner())
2656            .retain(|_, owner| owner != module_id);
2657        let removed = self
2658            .modules
2659            .lock()
2660            .unwrap_or_else(|poisoned| poisoned.into_inner())
2661            .remove(module_id);
2662        self.configured_ids
2663            .lock()
2664            .unwrap_or_else(|poisoned| poisoned.into_inner())
2665            .remove(module_id);
2666        removed
2667    }
2668
2669    /// Remember a module removed by a non-preview rescan so route.open can
2670    /// distinguish that intentional removal from an unknown id.
2671    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2672        self.removal_tombstones
2673            .lock()
2674            .unwrap_or_else(|poisoned| poisoned.into_inner())
2675            .insert(module_id.to_string(), unix_ms_now());
2676    }
2677
2678    /// Return how long ago a rescan removed this module in milliseconds.
2679    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2680        self.removal_tombstones
2681            .lock()
2682            .unwrap_or_else(|poisoned| poisoned.into_inner())
2683            .get(module_id)
2684            .copied()
2685            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2686    }
2687
2688    /// Retire a reserved-id gate only after its module has left supervision.
2689    ///
2690    /// A retained gate has no live nonce (`None`), so releasing any other entry
2691    /// would weaken a currently configured or otherwise active reservation.
2692    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2693        if self.get(module_id).is_some() {
2694            return false;
2695        }
2696        let mut reserved_nonces = self
2697            .reserved_nonces
2698            .lock()
2699            .unwrap_or_else(|poisoned| poisoned.into_inner());
2700        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2701            return false;
2702        }
2703        reserved_nonces.remove(module_id);
2704        true
2705    }
2706
2707    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2708        Arc::clone(&self.operation_lock)
2709    }
2710}
2711
2712/// Process supervisor for subc-owned singleton modules.
2713#[derive(Debug, Clone)]
2714pub struct Supervisor {
2715    registry: Arc<Registry>,
2716    restart_policy: RestartPolicy,
2717    drain_timeout: Duration,
2718    connection_file_path: Option<PathBuf>,
2719    capture_logs_dir: Option<PathBuf>,
2720    forwarding: Option<Arc<ForwardingTable>>,
2721    process_liveness: Arc<SupervisorProcessLiveness>,
2722    supervisor_handle: Option<SupervisorHandle>,
2723    health: HealthConfig,
2724    daemon_start_clock: crate::clock::StartClock,
2725    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2726    spawn_events: SpawnEventFeed,
2727    provenance_probe: ExecutableIdentityProbe,
2728    /// Every process spawned through this supervisor (and its clones) and not
2729    /// yet reaped, so daemon shutdown can end them.
2730    child_roster: ChildRoster,
2731    #[cfg(target_os = "linux")]
2732    cgroup_placement: Option<subc_cgroup::Placement>,
2733    #[cfg(test)]
2734    test_after_first_spawn: AfterFirstSpawnHook,
2735}
2736
2737/// A roster entry whose monitor and first process have not started yet.
2738/// Keeping the command receiver here lets discovery serve status immediately
2739/// while start/stop requests wait until bootstrap has published the connection file.
2740pub(crate) struct RegisteredConfiguredModule {
2741    spec: ModuleSpec,
2742    runtime: SupervisorRuntimeConfig,
2743    module: SupervisedModule,
2744    commands: mpsc::Receiver<SupervisorCommand>,
2745}
2746
2747/// Test-only hook run on the path that takes on a new module, right after its
2748/// first `spawn_child` returns (the process exists and could already be
2749/// registering) and before that process is handed to the module's supervise
2750/// loop and put on the roster. Lets a test observe what a fast child would see
2751/// in that window without racing a real one.
2752#[cfg(test)]
2753#[derive(Clone, Default)]
2754struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2755
2756#[cfg(test)]
2757type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2758
2759#[cfg(test)]
2760impl fmt::Debug for AfterFirstSpawnHook {
2761    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2762        f.write_str("AfterFirstSpawnHook")
2763    }
2764}
2765
2766#[cfg(test)]
2767impl AfterFirstSpawnHook {
2768    fn run(&self, module_id: &str) {
2769        if let Some(hook) = &self.0 {
2770            hook(module_id);
2771        }
2772    }
2773}
2774
2775impl Supervisor {
2776    #[cfg(test)]
2777    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2778        let supervisor = Self::new(registry, policy);
2779        #[cfg(target_os = "macos")]
2780        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2781        supervisor
2782    }
2783    /// Verify the trampoline once when configured. Missing private OS support
2784    /// refuses every macOS launch by name but does not stop the daemon's control
2785    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2786    /// entry point; the library must not exec an arbitrary hosting program.
2787    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2788        let path = path.into();
2789        #[cfg(target_os = "macos")]
2790        {
2791            let result = probe_privacy_trampoline(&path).map(|()| path);
2792            if let Err(cause) = &result {
2793                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2794            }
2795            self.child_roster.set_privacy_trampoline(result);
2796        }
2797        #[cfg(not(target_os = "macos"))]
2798        let _ = path;
2799        self
2800    }
2801    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2802    /// capability-requirement log to read.
2803    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2804        self.child_roster.shutdown_flag()
2805    }
2806
2807    /// The first step of an announced daemon shutdown, before the notice and
2808    /// before any connection is closed.
2809    ///
2810    /// Sets the daemon-shutdown flag first: from here on no module is
2811    /// respawned (crash restart, operator restart, or swap), and every child
2812    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2813    /// the module exits on the EOF this shutdown gives it or is signalled by a
2814    /// service manager that kills the whole cgroup. Then writes the journal's
2815    /// shutdown marker, which records the instant and closes this daemon
2816    /// incarnation's stretch of the journal.
2817    #[cfg(unix)]
2818    pub(crate) fn begin_daemon_shutdown(&self) {
2819        self.child_roster.close();
2820        if let Some(journal) = &self.terminal_journal {
2821            journal.stamp_shutdown();
2822        }
2823    }
2824
2825    /// Announce a cut while established connections can still carry replies.
2826    /// These budgets promise notice and a bounded wait, not child completion;
2827    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2828    #[cfg(unix)]
2829    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2830        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2831        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2832        let Some(forwarding) = &self.forwarding else {
2833            return Ok(());
2834        };
2835        let notice_started = Instant::now();
2836        let modules = forwarding
2837            .begin_daemon_drain()
2838            .map_err(SuperviseError::Forwarding)?;
2839        let deadline_ms =
2840            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2841        let mut notices = tokio::task::JoinSet::new();
2842        let mut drains = Vec::new();
2843        for (module_id, endpoint) in modules {
2844            let Some(target) = forwarding
2845                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2846                .map_err(SuperviseError::Forwarding)?
2847            else {
2848                continue;
2849            };
2850            let routes = forwarding
2851                .endpoint_routes(target.endpoint)
2852                .map_err(SuperviseError::Forwarding)?;
2853            // Restart allows deployed consumers to reopen after the new daemon
2854            // appears. The wire reason stays `restart`; what tells a daemon cut
2855            // apart from a module restart afterwards is the terminal record
2856            // itself, whose disposition is `daemon_shutdown` for every exit
2857            // observed once `begin_daemon_shutdown` has run.
2858            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2859                reason: RouteCloseReason::Restart,
2860                deadline_ms,
2861            })
2862            .expect("module draining serializes");
2863            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2864            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2865            for route in routes {
2866                let client = route.goodbye_target;
2867                if let Some((_, channels)) = clients
2868                    .iter_mut()
2869                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2870                {
2871                    channels.push(client.channel);
2872                } else {
2873                    let channel = client.channel;
2874                    clients.push((client, vec![channel]));
2875                }
2876            }
2877            for (client, mut channels) in clients {
2878                channels.sort_unstable();
2879                channels.dedup();
2880                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2881                    module_id: module_id.clone(),
2882                    channels,
2883                    reason: RouteCloseReason::Restart,
2884                })
2885                .expect("route closing serializes");
2886                recipients.push((client.sink, client.negotiated_ver, closing));
2887            }
2888            for (sink, version, body) in recipients {
2889                notices.spawn(async move {
2890                    let frame = Frame::build_with_version(
2891                        version,
2892                        FrameType::Push,
2893                        control_flags(),
2894                        0,
2895                        0,
2896                        0,
2897                        body,
2898                    )
2899                    .expect("bounded lifecycle notice frame builds");
2900                    sink.send_flushed(frame).await
2901                });
2902            }
2903            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2904            drains.push((module_id, target.endpoint, gauges));
2905        }
2906        // A quiet forwarding table is not proof that queued notices reached the
2907        // socket. Wait for writer flush acknowledgements before testing quiescence.
2908        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2909        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2910            if !matches!(result, Ok(Ok(()))) {
2911                warn!(?result, "daemon shutdown notice delivery failed");
2912            }
2913        }
2914        notices.abort_all();
2915        let notice_elapsed = notice_started.elapsed();
2916        let drain_started = Instant::now();
2917        let deadline = drain_started + DRAIN_BUDGET;
2918        let mut waits = tokio::task::JoinSet::new();
2919        for (module_id, endpoint, gauges) in drains {
2920            let forwarding = Arc::clone(forwarding);
2921            let mut runtime = self.runtime_config();
2922            runtime.health.cadence = Duration::from_millis(100);
2923            waits.spawn(async move {
2924                let wait_result = wait_for_forwarding_quiescence(
2925                    &forwarding,
2926                    &module_id,
2927                    &runtime,
2928                    endpoint,
2929                    deadline,
2930                    &gauges,
2931                    DrainScope::Endpoint(endpoint),
2932                    &mut DrainGaugeTrace::new(&gauges),
2933                )
2934                .await;
2935                // Store the measurement before the task returns: quiescence
2936                // may be reached just as the deadline stops the join loop from
2937                // collecting results. Unfinished or failed waits report false.
2938                forwarding
2939                    .record_daemon_drain_result(
2940                        endpoint,
2941                        drained_after_quiescence_wait(&wait_result),
2942                    )
2943                    .map_err(SuperviseError::Forwarding)?;
2944                wait_result
2945            });
2946        }
2947        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2948            if !matches!(result, Ok(Ok(true))) {
2949                warn!(?result, "daemon shutdown drain did not reach quiescence");
2950            }
2951        }
2952        // Measure only the forwarding drain, not child teardown or the caller's
2953        // signal delivery and process-reaping delays.
2954        info!(
2955            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2956            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2957            "daemon shutdown forwarding drain complete"
2958        );
2959        Ok(())
2960    }
2961
2962    /// The last step of an announced daemon shutdown, after the notice and the
2963    /// drain: send every registered module a module GOODBYE, the same planned
2964    /// stop signal `ck module stop` gives, then close every connection so each
2965    /// subc module sees EOF and starts its own teardown, then end every
2966    /// supervised child that has not exited
2967    /// by its own deadline (its drain budget, capped). Modules lead their own
2968    /// process groups, so a
2969    /// service manager's group kill no longer reaches them; without this a
2970    /// child that does not stop on EOF (every `protocol: "none"` child, which
2971    /// has no connection) would outlive the daemon. Every wait is bounded (see
2972    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2973    #[cfg(unix)]
2974    pub(crate) async fn end_children_for_daemon_shutdown(
2975        &self,
2976        already_escalated: bool,
2977        escalate: impl std::future::Future<Output = ()>,
2978    ) {
2979        tokio::pin!(escalate);
2980        let mut escalated = already_escalated;
2981        if let Some(forwarding) = &self.forwarding {
2982            let reason = CloseReason::new(
2983                "daemon_shutdown",
2984                "the daemon is exiting after its shutdown notice and drain",
2985            );
2986            if escalated {
2987                // The operator asked to stop waiting: queue the GOODBYEs but
2988                // do not wait for them to be written.
2989                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2990            } else {
2991                tokio::select! {
2992                    biased;
2993                    _ = escalate.as_mut() => {
2994                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2995                        escalated = true;
2996                    }
2997                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2998                }
2999            }
3000            let closed = forwarding.close_all_connections(&reason);
3001            debug!(closed, "closed established connections for daemon shutdown");
3002        }
3003        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3004        // already completed and must not be polled again; the child shutdown
3005        // wait is told it is escalated and gets a future that never fires.
3006        let escalated_here = escalated && !already_escalated;
3007        let remaining_escalate = async move {
3008            if escalated_here {
3009                std::future::pending::<()>().await;
3010            } else {
3011                escalate.await;
3012            }
3013        };
3014        crate::child_roster::end_children_for_daemon_shutdown(
3015            &self.child_roster,
3016            escalated,
3017            remaining_escalate,
3018        )
3019        .await;
3020    }
3021
3022    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3023        Self {
3024            registry,
3025            restart_policy,
3026            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3027            connection_file_path: None,
3028            capture_logs_dir: None,
3029            forwarding: None,
3030            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3031            supervisor_handle: None,
3032            health: HealthConfig::default(),
3033            daemon_start_clock: crate::clock::StartClock::capture(),
3034            terminal_journal: None,
3035            spawn_events: SpawnEventFeed::default(),
3036            provenance_probe: ExecutableIdentityProbe::default(),
3037            child_roster: ChildRoster::default(),
3038            #[cfg(target_os = "linux")]
3039            cgroup_placement: None,
3040            #[cfg(test)]
3041            test_after_first_spawn: AfterFirstSpawnHook::default(),
3042        }
3043    }
3044
3045    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3046        self.drain_timeout = drain_timeout;
3047        self
3048    }
3049
3050    /// The drain budget a module gets when its config entry sets none.
3051    pub(crate) fn default_drain_timeout(&self) -> Duration {
3052        self.drain_timeout
3053    }
3054
3055    pub fn with_process_liveness(
3056        mut self,
3057        process_liveness: Arc<SupervisorProcessLiveness>,
3058    ) -> Self {
3059        self.process_liveness = process_liveness;
3060        self
3061    }
3062
3063    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3064        self.connection_file_path = Some(connection_file_path.into());
3065        self
3066    }
3067
3068    /// Enables daemon-owned capture files for supervised stdout and stderr.
3069    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3070        self.capture_logs_dir = Some(logs_dir.into());
3071        self
3072    }
3073
3074    /// Names this daemon lifetime in spawn events, independently of whether a
3075    /// terminal journal is configured.
3076    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3077        // A millisecond start stamp can repeat after clock rollback or a rapid
3078        // restart. Use the connection file's random daemon_id instead: it already
3079        // identifies this daemon lifetime independently of the wall clock.
3080        self.spawn_events.configure_incarnation(daemon_incarnation);
3081        self
3082    }
3083
3084    /// Enables best-effort history shared by every supervised module. Without
3085    /// it, terminal history is kept only in each module's in-memory ring.
3086    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3087        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3088        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3089            path,
3090            daemon_incarnation,
3091        )));
3092        this
3093    }
3094
3095    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3096        self.forwarding = Some(forwarding);
3097        self
3098    }
3099
3100    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3101        self.spawn_events = supervisor_handle.spawn_events.clone();
3102        self.supervisor_handle = Some(supervisor_handle);
3103        self
3104    }
3105
3106    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3107        self.health = health;
3108        self
3109    }
3110
3111    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3112    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3113    /// record is kept.
3114    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3115        self.child_roster.record_to(path.into());
3116        self
3117    }
3118
3119    #[cfg(target_os = "linux")]
3120    pub fn with_cgroup_placement(
3121        mut self,
3122        cgroup_placement: Option<subc_cgroup::Placement>,
3123    ) -> Self {
3124        self.cgroup_placement = cgroup_placement;
3125        self
3126    }
3127
3128    /// Spawn `spec.program` and start monitoring it.
3129    ///
3130    /// The child is expected to parse `--subc <connection-file-path>`, read the
3131    /// TCP+key connection file, authenticate to the already-running listener, and
3132    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3133    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3134        validate_spec(&spec)?;
3135        self.establish_identity(&spec);
3136
3137        let runtime = self.runtime_config();
3138        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3139        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3140        let registration_watch = watch_registration(&self.registry, &spec);
3141        let spawned = spawn_child(
3142            &spec,
3143            runtime.connection_file_path.as_deref(),
3144            self.supervisor_handle.as_ref(),
3145            &runtime.stderr_ring,
3146            runtime.capture_logs_dir.as_deref(),
3147            &runtime.child_roster,
3148            #[cfg(target_os = "linux")]
3149            runtime.cgroup_placement.as_ref(),
3150        );
3151        #[cfg(test)]
3152        self.test_after_first_spawn.run(&spec.module_id);
3153        let mut child = match spawned {
3154            Ok(child) => child,
3155            Err(err) => {
3156                // Unlike the configured paths, a failed `spawn` leaves nothing
3157                // on the roster, so the module must not stay marked configured.
3158                self.abandon_unrostered(&spec.module_id);
3159                return Err(err);
3160            }
3161        };
3162        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3163        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3164        if let Some(spawn) = activity_spawn {
3165            spawn.complete(&snapshot);
3166        }
3167
3168        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3169    }
3170
3171    /// Make `spec`'s module count as configured, with its identity gates
3172    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3173    /// exists.
3174    ///
3175    /// Every path that takes on a new module calls this before `spawn_child`.
3176    /// The order is the point: the child can connect, register, sync its
3177    /// scopes and ask about them as soon as it is spawned, and the module is
3178    /// only put on the roster after `spawn_child` returns. Were the mark set
3179    /// with the roster entry, a fast child would see its own owner reported
3180    /// as not configured, and a scoped `route.open` in that window would be
3181    /// refused as terminal `scope_not_live` ("will never sync") instead of
3182    /// retryable `scope_not_synced`.
3183    fn establish_identity(&self, spec: &ModuleSpec) {
3184        if let Some(supervisor_handle) = &self.supervisor_handle {
3185            supervisor_handle.apply_identity_configuration(spec);
3186            supervisor_handle.mark_configured(&spec.module_id);
3187        }
3188    }
3189
3190    /// Take back [`Self::establish_identity`]'s configured mark when the
3191    /// module will not be put on the roster after all.
3192    fn abandon_unrostered(&self, module_id: &str) {
3193        if let Some(supervisor_handle) = &self.supervisor_handle {
3194            supervisor_handle.unmark_configured_unless_rostered(module_id);
3195        }
3196    }
3197
3198    /// Record a freshly spawned first process as running. On failure the
3199    /// module never reaches the roster, so its configured mark is taken back.
3200    fn mark_first_process_running(
3201        &self,
3202        spec: &ModuleSpec,
3203        runtime: &SupervisorRuntimeConfig,
3204        snapshot: &SharedSnapshot,
3205        child: &mut SupervisedChild,
3206    ) -> Result<(), SuperviseError> {
3207        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3208            self.abandon_unrostered(&spec.module_id);
3209            return Err(err);
3210        }
3211        self.process_liveness
3212            .track(spec.module_id.clone(), Arc::clone(snapshot));
3213        Ok(())
3214    }
3215
3216    /// Start supervising a module declared in daemon configuration.
3217    ///
3218    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3219    /// failures in the supervisor handle so operator-facing `supervisor.list`
3220    /// reflects every configured module while daemon startup continues.
3221    pub fn supervise_configured(
3222        &self,
3223        spec: ModuleSpec,
3224        enabled: bool,
3225    ) -> Result<SupervisedModule, SuperviseError> {
3226        self.supervise_configured_requested(
3227            spec,
3228            enabled,
3229            self.health.clone(),
3230            None,
3231            self.restart_policy,
3232            DEFAULT_REGISTER_TIMEOUT,
3233            None,
3234        )
3235    }
3236
3237    /// Supervise a configured module with its own health, drain, and crash
3238    /// budget. The restart policy is per-module because the config file is:
3239    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3240    /// module that is expensive to restart should not be forced onto the same
3241    /// budget as one that is cheap.
3242    pub fn supervise_configured_with_health(
3243        &self,
3244        spec: ModuleSpec,
3245        enabled: bool,
3246        health: HealthConfig,
3247        drain_timeout_ms: Option<u64>,
3248        restart_policy: RestartPolicy,
3249    ) -> Result<SupervisedModule, SuperviseError> {
3250        self.supervise_configured_requested(
3251            spec,
3252            enabled,
3253            health,
3254            drain_timeout_ms,
3255            restart_policy,
3256            DEFAULT_REGISTER_TIMEOUT,
3257            None,
3258        )
3259    }
3260
3261    #[allow(clippy::too_many_arguments)]
3262    pub(crate) fn supervise_configured_requested(
3263        &self,
3264        spec: ModuleSpec,
3265        enabled: bool,
3266        health: HealthConfig,
3267        drain_timeout_ms: Option<u64>,
3268        restart_policy: RestartPolicy,
3269        register_timeout: Duration,
3270        requester: Option<Principal>,
3271    ) -> Result<SupervisedModule, SuperviseError> {
3272        let registered = self.register_configured_with_health(
3273            spec,
3274            enabled,
3275            health,
3276            drain_timeout_ms,
3277            restart_policy,
3278            register_timeout,
3279        )?;
3280        let daemon_initiated = requester.is_none();
3281        self.start_registered_configured(registered, requester, daemon_initiated)
3282    }
3283
3284    /// Record a configured module as starting or disabled, without launching a
3285    /// process or running its monitor. Bootstrap registers the entire roster
3286    /// before publishing discovery, then activates it once children can read
3287    /// the connection file passed in their `--subc` argument.
3288    pub(crate) fn register_configured_with_health(
3289        &self,
3290        spec: ModuleSpec,
3291        enabled: bool,
3292        health: HealthConfig,
3293        drain_timeout_ms: Option<u64>,
3294        restart_policy: RestartPolicy,
3295        register_timeout: Duration,
3296    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3297        validate_spec(&spec)?;
3298        self.establish_identity(&spec);
3299
3300        let mut runtime = self.runtime_config();
3301        runtime.health = health;
3302        runtime.restart_policy = restart_policy;
3303        runtime.register_timeout = register_timeout;
3304        if let Some(ms) = drain_timeout_ms {
3305            runtime.drain_timeout = Duration::from_millis(ms);
3306            *runtime
3307                .effective_drain_timeout
3308                .lock()
3309                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3310        }
3311        let snapshot = if enabled {
3312            SupervisorSnapshot::starting()
3313        } else {
3314            SupervisorSnapshot::disabled()
3315        };
3316        let snapshot = {
3317            let mut snapshot = snapshot;
3318            if enabled {
3319                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3320                {
3321                    snapshot.waiting_for_previous_pid = Some(pid);
3322                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3323                }
3324            }
3325            snapshot
3326        };
3327        let (module, commands) =
3328            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3329        Ok(RegisteredConfiguredModule {
3330            spec,
3331            runtime,
3332            module,
3333            commands,
3334        })
3335    }
3336
3337    /// Activate a previously registered module. Queued control requests are
3338    /// served by the monitor after the initial configured spawn has settled.
3339    pub(crate) fn start_registered_configured(
3340        &self,
3341        registered: RegisteredConfiguredModule,
3342        requester: Option<Principal>,
3343        daemon_initiated: bool,
3344    ) -> Result<SupervisedModule, SuperviseError> {
3345        let RegisteredConfiguredModule {
3346            spec,
3347            mut runtime,
3348            module,
3349            commands,
3350        } = registered;
3351        runtime.start_requester = requester;
3352        runtime.recovery_is_daemon = daemon_initiated;
3353        let snapshot = &module.inner.snapshot;
3354        let should_spawn = {
3355            let state = lock_snapshot(snapshot)?;
3356            state.enabled && state.waiting_for_previous_pid.is_none()
3357        };
3358        let child = if should_spawn {
3359            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3360            let registration_watch = watch_registration(&self.registry, &spec);
3361            let spawned = spawn_child(
3362                &spec,
3363                runtime.connection_file_path.as_deref(),
3364                self.supervisor_handle.as_ref(),
3365                &runtime.stderr_ring,
3366                runtime.capture_logs_dir.as_deref(),
3367                &runtime.child_roster,
3368                #[cfg(target_os = "linux")]
3369                runtime.cgroup_placement.as_ref(),
3370            );
3371            #[cfg(test)]
3372            self.test_after_first_spawn.run(&spec.module_id);
3373            match spawned {
3374                Ok(mut child) => {
3375                    arm_registration_deadline(
3376                        &mut child,
3377                        runtime.register_timeout,
3378                        registration_watch,
3379                    );
3380                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3381                    if let Some(spawn) = activity_spawn {
3382                        spawn.complete(snapshot);
3383                    }
3384                    Some(child)
3385                }
3386                Err(err) => {
3387                    if runtime.health.critical {
3388                        error!(
3389                            module_id = %spec.module_id,
3390                            program = %spec.program.display(),
3391                            error = %err,
3392                            "critical configured module failed to spawn; marking failed and alerting"
3393                        );
3394                    } else {
3395                        error!(
3396                            module_id = %spec.module_id,
3397                            program = %spec.program.display(),
3398                            error = %err,
3399                            "configured module failed to spawn; marking failed and continuing"
3400                        );
3401                    }
3402                    fail_snapshot(
3403                        snapshot,
3404                        Some(&spec.module_id),
3405                        None,
3406                        &runtime.failure_context(daemon_initiated),
3407                    );
3408                    None
3409                }
3410            }
3411        } else {
3412            None
3413        };
3414        self.start_monitor(spec, runtime, &module, child, commands);
3415        Ok(module)
3416    }
3417
3418    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3419        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3420        SupervisorRuntimeConfig {
3421            registry: self.registry.clone(),
3422            recovery_is_daemon: true,
3423            start_requester: None,
3424            pending_activity_restart: false,
3425            provenance_probe: self.provenance_probe.clone(),
3426            scheduled_respawn: Arc::default(),
3427            deferred_reload_reply: Arc::default(),
3428            restart_policy: self.restart_policy,
3429            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3430            drain_timeout: self.drain_timeout,
3431            // Shared with this module's roster copy: daemon shutdown waits on
3432            // each child for the module's own drain budget, as resolved now.
3433            child_roster: self
3434                .child_roster
3435                .for_module(Arc::clone(&effective_drain_timeout)),
3436            effective_drain_timeout,
3437            default_drain_timeout: self.drain_timeout,
3438            health: self.health.clone(),
3439            connection_file_path: self.connection_file_path.clone(),
3440            capture_logs_dir: self.capture_logs_dir.clone(),
3441            forwarding: self.forwarding.clone(),
3442            supervisor_handle: self.supervisor_handle.clone(),
3443            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3444            terminal_ring: Arc::new(Mutex::new(
3445                TerminalRing::new(
3446                    TerminalRingConfig::default(),
3447                    self.daemon_start_clock.started_at_ms(),
3448                )
3449                .with_start_clock(self.daemon_start_clock)
3450                .with_journal(self.terminal_journal.clone())
3451                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3452            )),
3453            spawn_events: self.spawn_events.clone(),
3454            #[cfg(target_os = "linux")]
3455            cgroup_placement: self.cgroup_placement.clone(),
3456            #[cfg(test)]
3457            test_seed_stale_facts_before_enable_spawn: false,
3458            #[cfg(test)]
3459            test_reload_exit_record_gate: None,
3460        }
3461    }
3462
3463    fn supervised_module(
3464        &self,
3465        spec: ModuleSpec,
3466        runtime: SupervisorRuntimeConfig,
3467        snapshot: SharedSnapshot,
3468        child: Option<SupervisedChild>,
3469    ) -> SupervisedModule {
3470        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3471        self.start_monitor(spec, runtime, &module, child, commands);
3472        module
3473    }
3474
3475    fn register_module(
3476        &self,
3477        spec: &ModuleSpec,
3478        runtime: &SupervisorRuntimeConfig,
3479        snapshot: SharedSnapshot,
3480    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3481        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3482            spec: spec.clone(),
3483            health: runtime.health.clone(),
3484        }));
3485        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3486        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3487        // The module's OWN policy, which may be its per-module config rather than
3488        // the supervisor-wide one; status must report the budget the supervise
3489        // loop actually enforces.
3490        let restart_policy = runtime.restart_policy;
3491        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3492        let (tx, rx) = mpsc::channel(4);
3493        let module_id = spec.module_id.clone();
3494        let module = SupervisedModule {
3495            inner: Arc::new(SupervisedModuleInner {
3496                module_id: module_id.clone(),
3497                registry: Arc::clone(&self.registry),
3498                snapshot,
3499                configuration,
3500                stderr_ring,
3501                terminal_ring,
3502                commands: tx,
3503                monitor: Mutex::new(None),
3504                restart_policy,
3505                effective_drain_timeout,
3506                provenance_probe: self.provenance_probe.clone(),
3507            }),
3508        };
3509        // Identity gates and the configured mark must precede the roster entry,
3510        // so a client never sees a module without its admission policy.
3511        if let Some(supervisor_handle) = &self.supervisor_handle {
3512            supervisor_handle.insert(module.clone());
3513        }
3514        (module, rx)
3515    }
3516
3517    fn start_monitor(
3518        &self,
3519        spec: ModuleSpec,
3520        runtime: SupervisorRuntimeConfig,
3521        module: &SupervisedModule,
3522        child: Option<SupervisedChild>,
3523        commands: mpsc::Receiver<SupervisorCommand>,
3524    ) {
3525        let monitor = tokio::spawn(supervise_loop(
3526            spec,
3527            runtime,
3528            Arc::clone(&self.registry),
3529            Arc::clone(&self.process_liveness),
3530            Arc::clone(&module.inner.snapshot),
3531            child,
3532            commands,
3533        ));
3534        *module
3535            .inner
3536            .monitor
3537            .lock()
3538            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3539    }
3540}
3541
3542impl Default for Supervisor {
3543    fn default() -> Self {
3544        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3545    }
3546}
3547
3548/// Handle to one supervised child process.
3549#[derive(Clone)]
3550pub struct SupervisedModule {
3551    inner: Arc<SupervisedModuleInner>,
3552}
3553
3554struct SupervisedModuleInner {
3555    module_id: String,
3556    registry: Arc<Registry>,
3557    snapshot: SharedSnapshot,
3558    configuration: Arc<Mutex<SupervisedConfiguration>>,
3559    stderr_ring: Arc<Mutex<StderrRing>>,
3560    terminal_ring: Arc<Mutex<TerminalRing>>,
3561    commands: mpsc::Sender<SupervisorCommand>,
3562    monitor: Mutex<Option<JoinHandle<()>>>,
3563    /// Copied from the supervisor's runtime config at spawn so `status()` can
3564    /// report the restart budget without reaching back into the supervisor. The
3565    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3566    restart_policy: RestartPolicy,
3567    effective_drain_timeout: Arc<Mutex<Duration>>,
3568    provenance_probe: ExecutableIdentityProbe,
3569}
3570
3571impl fmt::Debug for SupervisedModule {
3572    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3573        f.debug_struct("SupervisedModule")
3574            .field("module_id", &self.inner.module_id)
3575            .field("status", &self.status())
3576            .finish_non_exhaustive()
3577    }
3578}
3579
3580impl SupervisedModule {
3581    pub fn module_id(&self) -> &str {
3582        &self.inner.module_id
3583    }
3584
3585    /// Test-only: put one probe miss on the streak, the way
3586    /// `handle_health_probe_failure` does, so tests can assert what a later
3587    /// event does to the streak without driving the whole probe loop.
3588    #[cfg(test)]
3589    pub(crate) fn record_health_probe_failure_for_test(
3590        &self,
3591        detail: &str,
3592    ) -> Result<(), SuperviseError> {
3593        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3594            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3595            state.health.detail = Some(detail.to_string());
3596        })
3597    }
3598
3599    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3600        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3601    }
3602
3603    pub(crate) fn registration_end_reason(
3604        &self,
3605    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3606        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3607        Ok(match snapshot.state {
3608            ModuleState::Draining if snapshot.draining_to_replace => {
3609                Some(RegistrationEndReason::SupervisorRestart)
3610            }
3611            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3612            ModuleState::Restarting if snapshot.draining_to_replace => {
3613                Some(RegistrationEndReason::SupervisorRestart)
3614            }
3615            _ => None,
3616        })
3617    }
3618
3619    /// The module's retained stderr, newest lines last.
3620    ///
3621    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3622    /// module, `supervisor.list` renders every module, and putting it in the
3623    /// shared snapshot would make each status read carry a payload almost nobody
3624    /// asked for. Callers that want the text ask for it.
3625    pub fn stderr_tail(
3626        &self,
3627        max_lines: Option<usize>,
3628        max_bytes: Option<usize>,
3629    ) -> StderrTailSnapshot {
3630        self.inner
3631            .stderr_ring
3632            .lock()
3633            .unwrap_or_else(|poisoned| poisoned.into_inner())
3634            .snapshot(max_lines, max_bytes)
3635    }
3636
3637    /// The module's bounded terminal history, oldest retained exit first.
3638    ///
3639    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3640    /// daemon whose in-memory history was necessarily reset.
3641    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3642        self.inner
3643            .terminal_ring
3644            .lock()
3645            .unwrap_or_else(|poisoned| poisoned.into_inner())
3646            .snapshot()
3647    }
3648
3649    /// Retained observations from the current ring and all journal generations.
3650    ///
3651    /// Blocking: this reads the journal files. Async callers use
3652    /// [`Self::read_durable_terminal_history`].
3653    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3654        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3655    }
3656
3657    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3658    /// read (up to every retained generation) never occupies a runtime worker.
3659    /// Fails only if the blocking task could not finish (runtime shutdown or a
3660    /// panic in the read).
3661    pub(crate) async fn read_durable_terminal_history(
3662        &self,
3663    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3664        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3665        let module_id = self.inner.module_id.clone();
3666        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3667            .await
3668    }
3669
3670    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3671        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3672            .map(|(status, _)| status)
3673    }
3674
3675    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3676        let state = lock_snapshot(&self.inner.snapshot)?;
3677        Ok(state
3678            .failed_retry
3679            .pending
3680            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3681            .map(|retry| {
3682                unix_ms_now().saturating_add(
3683                    u64::try_from(
3684                        retry
3685                            .deadline
3686                            .saturating_duration_since(Instant::now())
3687                            .as_millis(),
3688                    )
3689                    .unwrap_or(u64::MAX),
3690                )
3691            }))
3692    }
3693
3694    pub(crate) fn record_deliberate_severance(
3695        &self,
3696        identity: ProcessIdentity,
3697    ) -> Result<bool, SuperviseError> {
3698        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3699        if snapshot.pid != Some(identity.pid)
3700            || snapshot.process_start_time != Some(identity.start_time)
3701        {
3702            return Ok(false);
3703        }
3704        snapshot.deliberate_severance = Some(identity);
3705        Ok(true)
3706    }
3707
3708    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3709    ///
3710    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3711    /// does not produce reader-observability logs.
3712    pub(crate) fn status_for_control(
3713        &self,
3714        caller: &'static str,
3715    ) -> Result<ModuleStatus, SuperviseError> {
3716        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3717            .map(|(status, _)| status)
3718    }
3719
3720    fn status_with_snapshot_lock(
3721        &self,
3722        snapshot: &SharedSnapshot,
3723        caller: Option<&'static str>,
3724    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3725        let mut guard = match caller {
3726            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3727            None => lock_snapshot(snapshot)?,
3728        };
3729        // Read the budget through the pruning path so a reader sees the same
3730        // in-window count the restart decision would use, not a stale total.
3731        let restart_count =
3732            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3733        let snapshot = guard.clone();
3734        drop(guard);
3735        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3736            SuperviseError::StatePoisoned {
3737                module_id: Some(self.inner.module_id.clone()),
3738            }
3739        })?;
3740        let registration_active = self
3741            .inner
3742            .registry
3743            .get_module(&self.inner.module_id)
3744            .map_err(SuperviseError::Registry)?
3745            .is_some();
3746        let protocol = snapshot
3747            .spawned_protocol
3748            .unwrap_or(self.declared_protocol()?);
3749        let running_process =
3750            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3751        // Registration is the difference between the two protocols and the only
3752        // one: a subc module that has not registered cannot serve a request even
3753        // though its process is up, and a `none` module never registers at all,
3754        // so requiring it there would pin `live` to false for the whole life of
3755        // a perfectly healthy process.
3756        let live = match protocol {
3757            ModuleProtocol::Subc => running_process && registration_active,
3758            ModuleProtocol::None => running_process,
3759        };
3760
3761        Ok((
3762            ModuleStatus {
3763                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3764                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3765                module_id: self.inner.module_id.clone(),
3766                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3767                state: snapshot.state,
3768                enabled: snapshot.enabled,
3769                process_alive: snapshot.process_alive,
3770                registration_active,
3771                protocol,
3772                live,
3773                restart_count,
3774                lifetime_restarts: snapshot.lifetime_restarts,
3775                spawn_generation: snapshot.spawn_generation,
3776                max_restarts: self.inner.restart_policy.max_restarts,
3777                restart_window: self.inner.restart_policy.window,
3778                drain_timeout,
3779                restart_backoff: self.inner.restart_policy.backoff,
3780                restart_max_backoff: self.inner.restart_policy.max_backoff,
3781                pid: snapshot.reported_pid(),
3782                spawned_at_ms: snapshot.spawned_at_ms,
3783                spawned_from: snapshot.spawned_from,
3784                launch_spec_digest: snapshot.launch_spec_digest,
3785                process_start_time: snapshot.process_start_time,
3786                last_exit: snapshot.last_exit,
3787                health: snapshot.health,
3788            },
3789            snapshot.spawned_file_identity.clone(),
3790        ))
3791    }
3792
3793    #[cfg(test)]
3794    pub(crate) fn hold_snapshot_for_test(
3795        &self,
3796        acquired: std::sync::mpsc::Sender<()>,
3797        hold: Duration,
3798    ) -> std::thread::JoinHandle<()> {
3799        let snapshot = Arc::clone(&self.inner.snapshot);
3800        std::thread::spawn(move || {
3801            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3802            acquired
3803                .send(())
3804                .expect("test receiver waits for snapshot lock");
3805            std::thread::sleep(hold);
3806        })
3807    }
3808
3809    /// The status and the running-image check for `supervisor.provenance`,
3810    /// taken from one status read. The exec acknowledgement can land between
3811    /// two separate reads, and the reply would then pair "no pid yet" with an
3812    /// image observed after the module started, which describes no single
3813    /// moment.
3814    pub(crate) async fn status_and_running_image_agreement(
3815        &self,
3816    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3817        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3818        let image = self
3819            .inner
3820            .provenance_probe
3821            .observe(
3822                status.pid,
3823                status.spawned_from.as_deref(),
3824                identity,
3825                status.process_start_time,
3826            )
3827            .await;
3828        Ok((status, image))
3829    }
3830
3831    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3832        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3833            Ok(snapshot) => snapshot.clone(),
3834            Err(_) => {
3835                return subc_control::RunningImageAgreement::Unavailable {
3836                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3837                };
3838            }
3839        };
3840        self.inner
3841            .provenance_probe
3842            .observe(
3843                snapshot.reported_pid(),
3844                snapshot.spawned_from.as_deref(),
3845                snapshot.spawned_file_identity.clone(),
3846                snapshot.process_start_time,
3847            )
3848            .await
3849    }
3850
3851    /// Memory and CPU time of the module's current process, read now. Only the
3852    /// process the supervisor spawned is read, not processes it has started.
3853    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3854        #[cfg(windows)]
3855        {
3856            let image = lock_snapshot(&self.inner.snapshot)
3857                .ok()
3858                .and_then(|snapshot| {
3859                    snapshot
3860                        .spawned_file_identity
3861                        .as_ref()
3862                        .and_then(|identity| identity.windows_image.clone())
3863                });
3864            match image {
3865                Some(image) => {
3866                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3867                }
3868                None => subc_control::ChildResourceUsage::Unavailable {
3869                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3870                },
3871            }
3872        }
3873        #[cfg(not(windows))]
3874        {
3875            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3876                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3877                Err(_) => {
3878                    return subc_control::ChildResourceUsage::Unavailable {
3879                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3880                    }
3881                }
3882            };
3883            crate::child_resources::read(pid, start_time)
3884        }
3885    }
3886
3887    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3888        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3889        Ok(match snapshot.state {
3890            ModuleState::Restarting => true,
3891            ModuleState::Failed | ModuleState::Disabled => false,
3892            _ if snapshot.enabled && snapshot.respawn_pending => true,
3893            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3894        })
3895    }
3896
3897    #[cfg(test)]
3898    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3899        self.is_warming_with_snapshot_lock(None)
3900    }
3901
3902    pub(crate) fn is_warming_for_control(
3903        &self,
3904        caller: &'static str,
3905    ) -> Result<bool, SuperviseError> {
3906        self.is_warming_with_snapshot_lock(Some(caller))
3907    }
3908
3909    fn is_warming_with_snapshot_lock(
3910        &self,
3911        caller: Option<&'static str>,
3912    ) -> Result<bool, SuperviseError> {
3913        let snapshot = match caller {
3914            Some(caller) => {
3915                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3916            }
3917            None => lock_snapshot(&self.inner.snapshot)?,
3918        }
3919        .clone();
3920        Ok(matches!(
3921            snapshot.state,
3922            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3923        ))
3924    }
3925
3926    /// Drain the module and stop monitoring it.
3927    pub async fn drain(&self) -> Result<(), SuperviseError> {
3928        self.stop().await
3929    }
3930
3931    pub(crate) async fn retire_requested(
3932        &self,
3933        requester: Option<Principal>,
3934    ) -> Result<(), SuperviseError> {
3935        match self.state()? {
3936            ModuleState::Stopped => return Ok(()),
3937            ModuleState::Failed
3938                if lock_snapshot(&self.inner.snapshot)?
3939                    .failed_retry
3940                    .pending
3941                    .is_none() =>
3942            {
3943                return Ok(())
3944            }
3945            ModuleState::Failed
3946            | ModuleState::Starting
3947            | ModuleState::Running
3948            | ModuleState::Unresponsive
3949            | ModuleState::Restarting
3950            | ModuleState::Draining
3951            | ModuleState::Disabled => {}
3952        }
3953
3954        let (reply_tx, reply_rx) = oneshot::channel();
3955        self.inner
3956            .commands
3957            .send(SupervisorCommand::Retire {
3958                requester,
3959                reply: reply_tx,
3960            })
3961            .await
3962            .map_err(|_| SuperviseError::CommandClosed {
3963                module_id: self.inner.module_id.clone(),
3964            })?;
3965        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3966            module_id: self.inner.module_id.clone(),
3967        })?
3968    }
3969
3970    pub async fn stop(&self) -> Result<(), SuperviseError> {
3971        self.stop_requested(None).await
3972    }
3973
3974    pub(crate) async fn stop_requested(
3975        &self,
3976        requester: Option<Principal>,
3977    ) -> Result<(), SuperviseError> {
3978        match self.state()? {
3979            ModuleState::Stopped => return Ok(()),
3980            ModuleState::Failed
3981                if lock_snapshot(&self.inner.snapshot)?
3982                    .failed_retry
3983                    .pending
3984                    .is_none() =>
3985            {
3986                return Ok(())
3987            }
3988            ModuleState::Failed
3989            | ModuleState::Starting
3990            | ModuleState::Running
3991            | ModuleState::Unresponsive
3992            | ModuleState::Restarting
3993            | ModuleState::Draining
3994            | ModuleState::Disabled => {}
3995        }
3996
3997        let (reply_tx, reply_rx) = oneshot::channel();
3998        self.inner
3999            .commands
4000            .send(SupervisorCommand::Drain {
4001                requester,
4002                reply: reply_tx,
4003            })
4004            .await
4005            .map_err(|_| SuperviseError::CommandClosed {
4006                module_id: self.inner.module_id.clone(),
4007            })?;
4008        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4009            module_id: self.inner.module_id.clone(),
4010        })?
4011    }
4012
4013    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4014        self.restart_requested(drain_timeout_ms, None).await
4015    }
4016
4017    pub(crate) async fn restart_requested(
4018        &self,
4019        drain_timeout_ms: Option<u64>,
4020        requester: Option<Principal>,
4021    ) -> Result<(), SuperviseError> {
4022        let queued_at = Instant::now();
4023        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4024        let (reply_tx, reply_rx) = oneshot::channel();
4025        self.inner
4026            .commands
4027            .send(SupervisorCommand::Restart {
4028                requester,
4029                drain_timeout_ms,
4030                received_at_generation,
4031                queued_at,
4032                reply: reply_tx,
4033            })
4034            .await
4035            .map_err(|_| SuperviseError::CommandClosed {
4036                module_id: self.inner.module_id.clone(),
4037            })?;
4038        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4039            module_id: self.inner.module_id.clone(),
4040        })?
4041    }
4042
4043    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
4044    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4045    /// process then drains in the background of the supervise loop) or has
4046    /// failed, leaving the old process serving.
4047    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4048        self.swap_requested(ready_timeout, None).await
4049    }
4050
4051    pub(crate) async fn swap_requested(
4052        &self,
4053        ready_timeout: Option<Duration>,
4054        requester: Option<Principal>,
4055    ) -> Result<(), SuperviseError> {
4056        let (reply_tx, reply_rx) = oneshot::channel();
4057        self.inner
4058            .commands
4059            .send(SupervisorCommand::Swap {
4060                requester,
4061                ready_timeout,
4062                reply: reply_tx,
4063            })
4064            .await
4065            .map_err(|_| SuperviseError::CommandClosed {
4066                module_id: self.inner.module_id.clone(),
4067            })?;
4068        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4069            module_id: self.inner.module_id.clone(),
4070        })?
4071    }
4072
4073    pub async fn reload(&self) -> Result<(), SuperviseError> {
4074        self.reload_requested(None).await
4075    }
4076
4077    pub(crate) async fn reload_requested(
4078        &self,
4079        requester: Option<Principal>,
4080    ) -> Result<(), SuperviseError> {
4081        let (reply_tx, reply_rx) = oneshot::channel();
4082        self.inner
4083            .commands
4084            .send(SupervisorCommand::Reload {
4085                requester,
4086                reply: reply_tx,
4087            })
4088            .await
4089            .map_err(|_| SuperviseError::CommandClosed {
4090                module_id: self.inner.module_id.clone(),
4091            })?;
4092        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4093            module_id: self.inner.module_id.clone(),
4094        })?
4095    }
4096
4097    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4098        self.set_enabled_requested(enabled, None).await
4099    }
4100
4101    pub(crate) async fn set_enabled_requested(
4102        &self,
4103        enabled: bool,
4104        requester: Option<Principal>,
4105    ) -> Result<bool, SuperviseError> {
4106        let (reply_tx, reply_rx) = oneshot::channel();
4107        self.inner
4108            .commands
4109            .send(SupervisorCommand::SetEnabled {
4110                requester,
4111                enabled,
4112                reply: reply_tx,
4113            })
4114            .await
4115            .map_err(|_| SuperviseError::CommandClosed {
4116                module_id: self.inner.module_id.clone(),
4117            })?;
4118        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4119            module_id: self.inner.module_id.clone(),
4120        })?
4121    }
4122
4123    /// The current process's protocol, or the configured protocol when down.
4124    /// A rescan stores the next launch spec without changing how an existing
4125    /// process registers, serves routes, is probed, or exits.
4126    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4127        let configured = self
4128            .inner
4129            .configuration
4130            .lock()
4131            .map_err(|_| SuperviseError::StatePoisoned {
4132                module_id: Some(self.inner.module_id.clone()),
4133            })?
4134            .spec
4135            .protocol;
4136        let state = lock_snapshot(&self.inner.snapshot)?;
4137        Ok(state.spawned_protocol.unwrap_or(configured))
4138    }
4139
4140    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4141        let configuration =
4142            self.inner
4143                .configuration
4144                .lock()
4145                .map_err(|_| SuperviseError::StatePoisoned {
4146                    module_id: Some(self.inner.module_id.clone()),
4147                })?;
4148        Ok((configuration.spec.clone(), configuration.health.clone()))
4149    }
4150
4151    /// Replace this module's launch spec, keeping its health and drain policy,
4152    /// the way a rescan does for a changed config entry. The running process is
4153    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4154    #[cfg(any(test, feature = "test-support"))]
4155    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4156        let (_, health) = self.configuration()?;
4157        let drain_timeout_ms = u64::try_from(
4158            self.inner
4159                .effective_drain_timeout
4160                .lock()
4161                .unwrap_or_else(|poisoned| poisoned.into_inner())
4162                .as_millis(),
4163        )
4164        .ok();
4165        self.update_configuration(spec, health, drain_timeout_ms)
4166            .await
4167    }
4168
4169    pub(crate) async fn update_configuration(
4170        &self,
4171        spec: ModuleSpec,
4172        health: HealthConfig,
4173        drain_timeout_ms: Option<u64>,
4174    ) -> Result<(), SuperviseError> {
4175        if spec.module_id != self.inner.module_id {
4176            return Err(SuperviseError::InvalidSpec {
4177                reason: "a supervised module's module_id cannot be changed".to_string(),
4178            });
4179        }
4180        validate_spec(&spec)?;
4181        let (reply_tx, reply_rx) = oneshot::channel();
4182        self.inner
4183            .commands
4184            .send(SupervisorCommand::UpdateConfiguration {
4185                spec: spec.clone(),
4186                health: health.clone(),
4187                drain_timeout_ms,
4188                reply: reply_tx,
4189            })
4190            .await
4191            .map_err(|_| SuperviseError::CommandClosed {
4192                module_id: self.inner.module_id.clone(),
4193            })?;
4194        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4195            module_id: self.inner.module_id.clone(),
4196        })?;
4197        let mut configuration =
4198            self.inner
4199                .configuration
4200                .lock()
4201                .map_err(|_| SuperviseError::StatePoisoned {
4202                    module_id: Some(self.inner.module_id.clone()),
4203                })?;
4204        configuration.spec = spec;
4205        configuration.health = health;
4206        Ok(())
4207    }
4208}
4209
4210impl Drop for SupervisedModuleInner {
4211    fn drop(&mut self) {
4212        let Ok(mut monitor) = self.monitor.lock() else {
4213            return;
4214        };
4215        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4216            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4217                state.state = ModuleState::Stopped;
4218                clear_current_process_facts(state);
4219            });
4220            monitor.abort();
4221        }
4222        let _ = monitor.take();
4223    }
4224}
4225
4226#[derive(Debug)]
4227enum SupervisorCommand {
4228    Drain {
4229        requester: Option<Principal>,
4230        reply: oneshot::Sender<Result<(), SuperviseError>>,
4231    },
4232    Retire {
4233        requester: Option<Principal>,
4234        reply: oneshot::Sender<Result<(), SuperviseError>>,
4235    },
4236    Restart {
4237        requester: Option<Principal>,
4238        /// Operator override for this one restart's drain budget, in ms. `None`
4239        /// uses the module's configured/default budget; `Some(0)` cuts
4240        /// immediately (wedge bounce: a stuck request never settles, so
4241        /// waiting only delays recovery).
4242        drain_timeout_ms: Option<u64>,
4243        /// The module's `spawn_generation` when the request was received, before
4244        /// it waited in the command queue. A queued restart whose module has
4245        /// since spawned a newer process is already satisfied (see the handler).
4246        received_at_generation: u64,
4247        /// When the request entered the command queue, so the handler can log
4248        /// how long it waited behind the loop's other work.
4249        queued_at: Instant,
4250        reply: oneshot::Sender<Result<(), SuperviseError>>,
4251    },
4252    Reload {
4253        requester: Option<Principal>,
4254        reply: oneshot::Sender<Result<(), SuperviseError>>,
4255    },
4256    SetEnabled {
4257        requester: Option<Principal>,
4258        enabled: bool,
4259        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4260    },
4261    UpdateConfiguration {
4262        spec: ModuleSpec,
4263        health: HealthConfig,
4264        /// Per-module drain override from the new config; `None` re-resolves to
4265        /// the supervisor-wide default.
4266        drain_timeout_ms: Option<u64>,
4267        reply: oneshot::Sender<()>,
4268    },
4269    Swap {
4270        requester: Option<Principal>,
4271        /// How long the candidate may take to register and declare itself
4272        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4273        ready_timeout: Option<Duration>,
4274        /// Answered at cutover or failure; the incumbent's drain follows.
4275        reply: oneshot::Sender<Result<(), SuperviseError>>,
4276    },
4277}
4278
4279#[derive(Debug)]
4280pub enum SuperviseError {
4281    InvalidSpec {
4282        reason: String,
4283    },
4284    Spawn {
4285        program: PathBuf,
4286        source: io::Error,
4287        cgroup_path: Option<PathBuf>,
4288    },
4289    Cgroup {
4290        module_id: String,
4291        source: io::Error,
4292    },
4293    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4294    /// than spawn a reserved module without its identity binding.
4295    LaunchNonce {
4296        reason: String,
4297    },
4298    Wait {
4299        module_id: String,
4300        source: io::Error,
4301    },
4302    Kill {
4303        module_id: String,
4304        source: io::Error,
4305    },
4306    Forwarding(ForwardingError),
4307    Registry(RegistryError),
4308    ReloadUnavailable {
4309        module_id: String,
4310        reason: String,
4311    },
4312    /// An operator restart/reload was requested for a module that is currently
4313    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4314    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4315    /// by a restart, so these commands are rejected instead of re-enabling it.
4316    Disabled {
4317        module_id: String,
4318    },
4319    ReloadFailed {
4320        module_id: String,
4321        reason: String,
4322    },
4323    RegistrationStillActive {
4324        module_id: String,
4325        waited: Duration,
4326    },
4327    StatePoisoned {
4328        module_id: Option<String>,
4329    },
4330    CommandClosed {
4331        module_id: String,
4332    },
4333    /// A restart or reload arrived while a swap's candidate was warming. The
4334    /// swap owns the module until it cuts over or fails; a stop or disable
4335    /// would have aborted it instead.
4336    SwapInProgress {
4337        module_id: String,
4338    },
4339    /// A swap was refused before anything was spawned.
4340    SwapRefused {
4341        module_id: String,
4342        reason: SwapRefusal,
4343    },
4344    /// A swap spawned a candidate and gave up on it. The candidate has been
4345    /// killed and its slot freed; the incumbent was left serving and was never
4346    /// drained, except in the one `CutoverLost` case described on that arm.
4347    SwapFailed {
4348        module_id: String,
4349        arm: SwapFailureArm,
4350        detail: String,
4351        /// How the candidate exited, when it exited on its own before the
4352        /// supervisor gave up on it.
4353        candidate_exit: Option<ExitReport>,
4354    },
4355}
4356
4357/// Why a swap was refused before a candidate was spawned.
4358#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4359pub enum SwapRefusal {
4360    /// The module's config does not declare `overlap: "safe"`.
4361    OverlapExclusive,
4362    /// The module is not registered, so there is no incumbent to keep serving
4363    /// and nothing a swap would improve on; a plain restart is the tool.
4364    NotRegistered,
4365    /// The module does not speak the subc wire, so a candidate could never
4366    /// register or declare itself ready.
4367    ProtocolNone,
4368    /// The supervisor lacks the forwarding table (to cut routes over) or the
4369    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4370    NotConfigured,
4371    /// A swap is already open for this module.
4372    AlreadySwapping,
4373}
4374
4375impl SwapRefusal {
4376    pub fn as_str(self) -> &'static str {
4377        match self {
4378            Self::OverlapExclusive => "overlap_exclusive",
4379            Self::NotRegistered => "not_registered",
4380            Self::ProtocolNone => "protocol_none",
4381            Self::NotConfigured => "not_configured",
4382            Self::AlreadySwapping => "already_swapping",
4383        }
4384    }
4385}
4386
4387/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4388/// serving and undrained; see `CutoverLost`.
4389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4390pub enum SwapFailureArm {
4391    /// The candidate process could not be started.
4392    SpawnFailed,
4393    /// The candidate did not register within the readiness budget.
4394    NeverRegistered,
4395    /// The candidate registered but did not declare itself ready in time.
4396    NeverReady,
4397    /// The candidate exited before cutover.
4398    CandidateExited,
4399    /// The candidate declared itself ready but failed its health probe.
4400    CandidateUnhealthy,
4401    /// An operator stop, disable or retire arrived while the candidate warmed.
4402    /// The candidate was killed and the operator's command then carried out on
4403    /// the incumbent.
4404    Interrupted,
4405    /// The candidate's connection closed at the moment of cutover. If it
4406    /// closed before forwarding moved, the incumbent is untouched. If it closed
4407    /// between the forwarding and registry halves of cutover, forwarding can no
4408    /// longer route to the incumbent, so the module is restarted plainly.
4409    CutoverLost,
4410}
4411
4412impl SwapFailureArm {
4413    pub fn as_str(self) -> &'static str {
4414        match self {
4415            Self::SpawnFailed => "spawn_failed",
4416            Self::NeverRegistered => "never_registered",
4417            Self::NeverReady => "never_ready",
4418            Self::CandidateExited => "candidate_exited",
4419            Self::CandidateUnhealthy => "candidate_unhealthy",
4420            Self::Interrupted => "interrupted",
4421            Self::CutoverLost => "cutover_lost",
4422        }
4423    }
4424}
4425
4426impl fmt::Display for SuperviseError {
4427    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4428        match self {
4429            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4430            Self::Spawn {
4431                program,
4432                source,
4433                cgroup_path: Some(cgroup_path),
4434            } => write!(
4435                f,
4436                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4437                cgroup_path.display(),
4438                program.display()
4439            ),
4440            Self::Spawn {
4441                program,
4442                source,
4443                cgroup_path: None,
4444            } => write!(
4445                f,
4446                "failed to spawn module '{}': {source}",
4447                program.display()
4448            ),
4449            Self::Cgroup { module_id, source } => {
4450                write!(
4451                    f,
4452                    "failed to prepare cgroup for module '{module_id}': {source}"
4453                )
4454            }
4455            Self::LaunchNonce { reason } => {
4456                write!(
4457                    f,
4458                    "failed to generate reserved-module launch nonce: {reason}"
4459                )
4460            }
4461            Self::Wait { module_id, source } => {
4462                write!(f, "failed to wait for module '{module_id}': {source}")
4463            }
4464            Self::Kill { module_id, source } => {
4465                write!(f, "failed to kill module '{module_id}': {source}")
4466            }
4467            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4468            Self::Registry(err) => write!(f, "registry error: {err}"),
4469            Self::ReloadUnavailable { module_id, reason } => {
4470                write!(f, "reload unavailable for module '{module_id}': {reason}")
4471            }
4472            Self::Disabled { module_id } => {
4473                write!(
4474                    f,
4475                    "module '{module_id}' is disabled; enable it before restart or reload"
4476                )
4477            }
4478            Self::ReloadFailed { module_id, reason } => {
4479                write!(f, "reload failed for module '{module_id}': {reason}")
4480            }
4481            Self::RegistrationStillActive { module_id, waited } => write!(
4482                f,
4483                "module '{module_id}' registration remained active after waiting {waited:?}"
4484            ),
4485            Self::StatePoisoned { module_id } => match module_id {
4486                Some(module_id) => {
4487                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4488                }
4489                None => write!(f, "supervisor state was poisoned"),
4490            },
4491            Self::CommandClosed { module_id } => {
4492                write!(
4493                    f,
4494                    "supervisor command channel for module '{module_id}' is closed"
4495                )
4496            }
4497            Self::SwapInProgress { module_id } => write!(
4498                f,
4499                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4500            ),
4501            Self::SwapRefused { module_id, reason } => match reason {
4502                SwapRefusal::OverlapExclusive => write!(
4503                    f,
4504                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4505                ),
4506                SwapRefusal::NotRegistered => write!(
4507                    f,
4508                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4509                ),
4510                SwapRefusal::ProtocolNone => write!(
4511                    f,
4512                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4513                ),
4514                SwapRefusal::NotConfigured => write!(
4515                    f,
4516                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4517                ),
4518                SwapRefusal::AlreadySwapping => {
4519                    write!(f, "module '{module_id}' is already being swapped")
4520                }
4521            },
4522            Self::SwapFailed {
4523                module_id,
4524                arm,
4525                detail,
4526                ..
4527            } => write!(
4528                f,
4529                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4530                arm.as_str()
4531            ),
4532        }
4533    }
4534}
4535
4536impl Error for SuperviseError {
4537    fn source(&self) -> Option<&(dyn Error + 'static)> {
4538        match self {
4539            Self::Spawn { source, .. }
4540            | Self::Cgroup { source, .. }
4541            | Self::Wait { source, .. }
4542            | Self::Kill { source, .. } => Some(source),
4543            Self::Forwarding(err) => Some(err),
4544            Self::Registry(err) => Some(err),
4545            Self::LaunchNonce { .. }
4546            | Self::InvalidSpec { .. }
4547            | Self::ReloadUnavailable { .. }
4548            | Self::Disabled { .. }
4549            | Self::ReloadFailed { .. }
4550            | Self::RegistrationStillActive { .. }
4551            | Self::StatePoisoned { .. }
4552            | Self::CommandClosed { .. }
4553            | Self::SwapInProgress { .. }
4554            | Self::SwapRefused { .. }
4555            | Self::SwapFailed { .. } => None,
4556        }
4557    }
4558}
4559
4560pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4561    if spec.module_id.trim().is_empty() {
4562        return Err(SuperviseError::InvalidSpec {
4563            reason: "module_id must not be empty".to_string(),
4564        });
4565    }
4566
4567    Ok(())
4568}
4569
4570#[derive(Debug, Default)]
4571struct HealthProbeRuntime {
4572    spawn_generation: Option<u64>,
4573    configured_health: Option<HealthConfig>,
4574    registered_connection: Option<crate::ConnectionId>,
4575    advertised: bool,
4576    next_probe_at: Option<Instant>,
4577    probe_index: u64,
4578}
4579
4580fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4581    lock_snapshot(snapshot)
4582        .ok()
4583        .and_then(|state| state.spawned_protocol)
4584        .unwrap_or(spec.protocol)
4585}
4586
4587impl HealthProbeRuntime {
4588    fn refresh_registration(
4589        &mut self,
4590        spec: &ModuleSpec,
4591        runtime: &SupervisorRuntimeConfig,
4592        registry: &Registry,
4593        snapshot: &SharedSnapshot,
4594    ) {
4595        let generation = lock_snapshot(snapshot)
4596            .ok()
4597            .map(|state| state.spawn_generation);
4598        if self.spawn_generation != generation {
4599            // A replacement cannot reuse a deadline that is already due for its
4600            // predecessor. HTTP children get a full cadence to start; wire
4601            // children still wait for their own health-advertising HELLO.
4602            *self = Self {
4603                spawn_generation: generation,
4604                ..Self::default()
4605            };
4606        }
4607        if self.configured_health.as_ref() != Some(&runtime.health) {
4608            self.configured_health = Some(runtime.health.clone());
4609            self.next_probe_at = None;
4610            self.registered_connection = None;
4611            self.probe_index = 0;
4612        }
4613        // A non-wire process never registers. Only an explicitly configured
4614        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4615        // health failure for that kind of process.
4616        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4617            self.registered_connection = None;
4618            self.advertised = runtime.health.http.is_some();
4619            if !self.advertised {
4620                self.next_probe_at = None;
4621                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622                    let unknown = ModuleHealthStatus::default();
4623                    if state.health != unknown {
4624                        state.health = unknown;
4625                    }
4626                });
4627            } else if self.next_probe_at.is_none() {
4628                self.next_probe_at = Some(
4629                    Instant::now()
4630                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4631                );
4632            }
4633            return;
4634        }
4635
4636        let registration = match registry.get_module(&spec.module_id) {
4637            Ok(registration) => registration,
4638            Err(err) => {
4639                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4640                self.advertised = false;
4641                self.next_probe_at = None;
4642                return;
4643            }
4644        };
4645
4646        let Some(registration) = registration else {
4647            self.registered_connection = None;
4648            self.advertised = false;
4649            self.next_probe_at = None;
4650            return;
4651        };
4652
4653        let advertised = registration
4654            .control_ops
4655            .iter()
4656            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4657        if !advertised {
4658            self.registered_connection = Some(registration.connection_id);
4659            self.advertised = false;
4660            self.next_probe_at = None;
4661            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4662                if state.health.status != SupervisorHealthStatus::Unknown
4663                    || state.health.consecutive_failures != 0
4664                    || state.health.last_probe_ms.is_some()
4665                    || state.health.detail.is_some()
4666                    || state.health.metrics.is_some()
4667                {
4668                    state.health.status = SupervisorHealthStatus::Unknown;
4669                    state.health.consecutive_failures = 0;
4670                    state.health.last_probe_ms = None;
4671                    state.health.detail = None;
4672                    state.health.metrics = None;
4673                }
4674            });
4675            return;
4676        }
4677
4678        let reregistered = self.registered_connection != Some(registration.connection_id);
4679        self.registered_connection = Some(registration.connection_id);
4680        self.advertised = true;
4681        if reregistered || self.next_probe_at.is_none() {
4682            self.probe_index = 0;
4683            self.next_probe_at = Some(
4684                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4685            );
4686            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4687                state.health.status = SupervisorHealthStatus::Unknown;
4688                state.health.consecutive_failures = 0;
4689                state.health.detail = None;
4690                state.health.metrics = None;
4691            });
4692        }
4693    }
4694
4695    fn wake_at(&self) -> Option<Instant> {
4696        if !self.advertised {
4697            return None;
4698        }
4699        self.next_probe_at
4700    }
4701
4702    fn due(&self) -> bool {
4703        self.advertised
4704            && self
4705                .next_probe_at
4706                .is_some_and(|next| Instant::now() >= next)
4707    }
4708
4709    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4710        self.probe_index = self.probe_index.wrapping_add(1);
4711        self.next_probe_at = Some(
4712            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4713        );
4714    }
4715}
4716
4717/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4718///
4719/// This was a struct with a single `message: String`, and every one of the
4720/// fifteen construction sites collapsed into it. Each site knows exactly what it
4721/// saw -- the lane is gone, the module did not answer in time, the module
4722/// answered with the wrong thing -- and `handle_health_probe_failure` then
4723/// treated all of them identically: increment a counter, compare to a threshold,
4724/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4725/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4726///
4727/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4728///
4729/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4730///   answer on it again.
4731/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4732///   AND with a perfectly healthy one that lost a CPU race -- which is what
4733///   happens under machine load, and is how this supervisor killed a healthy
4734///   module three times in one day.
4735/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4736///   Restarting on it is defensible, but it is not the silence case and should
4737///   never be counted as one.
4738/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4739///   anything, so it cannot be evidence about the module at all.
4740///
4741/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4742/// one that fires most often, and while every variant collapsed into one string
4743/// it carried the same weight as the strongest.
4744///
4745/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4746/// DESIGN and a reader stopping at it gets the build backwards: the restart
4747/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4748/// probes still increment the failure streak and drive escalation at the
4749/// threshold (see `is_proof_of_death` below for why that is deliberate and
4750/// what gates the change). Absence of evidence restarts modules today.
4751#[derive(Debug)]
4752enum HealthProbeEvidence {
4753    /// The module's control lane is gone. Proof of death.
4754    LaneDead,
4755    /// No reply within the deadline. Proves nothing about the module's state.
4756    NoAnswer,
4757    /// The module replied, but not with a usable health report. Proves it is alive.
4758    BadAnswer,
4759    /// The daemon could not ask. Says nothing about the module.
4760    Misconfigured,
4761}
4762
4763#[derive(Debug)]
4764struct HealthProbeError {
4765    evidence: HealthProbeEvidence,
4766    message: String,
4767}
4768
4769impl HealthProbeError {
4770    fn lane_dead(message: impl Into<String>) -> Self {
4771        Self::with(HealthProbeEvidence::LaneDead, message)
4772    }
4773
4774    fn no_answer(message: impl Into<String>) -> Self {
4775        Self::with(HealthProbeEvidence::NoAnswer, message)
4776    }
4777
4778    fn bad_answer(message: impl Into<String>) -> Self {
4779        Self::with(HealthProbeEvidence::BadAnswer, message)
4780    }
4781
4782    fn misconfigured(message: impl Into<String>) -> Self {
4783        Self::with(HealthProbeEvidence::Misconfigured, message)
4784    }
4785
4786    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4787        Self {
4788            evidence,
4789            message: message.into(),
4790        }
4791    }
4792
4793    /// Whether this observation is proof the module cannot serve.
4794    ///
4795    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4796    /// variant that fires under CPU starvation, and treating it as proof is the
4797    /// defect this enum exists to make impossible to reintroduce silently.
4798    ///
4799    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4800    /// to restart also needs a bound for the case it excludes -- a genuinely
4801    /// wedged module, alive but never answering -- and that bound must come from
4802    /// the distribution of real late-answer latencies, which nothing measures
4803    /// yet. Landing the classification first makes the later change a one-line
4804    /// decision against evidence that already exists, rather than two unproven
4805    /// changes at once.
4806    #[allow(dead_code)]
4807    fn is_proof_of_death(&self) -> bool {
4808        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4809    }
4810
4811    /// Short stable label for logs and the health snapshot.
4812    ///
4813    /// An operator reading `ck health` currently cannot tell "the module is gone"
4814    /// from "the module did not answer in five seconds", because both render as
4815    /// prose in the same field. These labels are what make the two
4816    /// distinguishable at a glance, and they are what a later restart-policy
4817    /// change will be argued from.
4818    fn label(&self) -> &'static str {
4819        match self.evidence {
4820            HealthProbeEvidence::LaneDead => "lane-dead",
4821            HealthProbeEvidence::NoAnswer => "no-answer",
4822            HealthProbeEvidence::BadAnswer => "bad-answer",
4823            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4824        }
4825    }
4826}
4827
4828impl fmt::Display for HealthProbeError {
4829    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4830        f.write_str(&self.message)
4831    }
4832}
4833
4834async fn run_health_probe_cycle(
4835    spec: &ModuleSpec,
4836    runtime: &SupervisorRuntimeConfig,
4837    registry: &Registry,
4838    process_liveness: &SupervisorProcessLiveness,
4839    snapshot: &SharedSnapshot,
4840    child: &mut Option<SupervisedChild>,
4841) {
4842    let now_ms = unix_ms_now();
4843    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4844        .then_some(runtime.health.http.as_deref())
4845        .flatten();
4846    let result = match http {
4847        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4848        None => probe_module_health(&spec.module_id, runtime, None).await,
4849    };
4850    match result {
4851        Ok(report) => {
4852            handle_health_report(
4853                spec,
4854                runtime,
4855                registry,
4856                process_liveness,
4857                snapshot,
4858                child,
4859                report,
4860                now_ms,
4861            )
4862            .await;
4863        }
4864        Err(err) => {
4865            if http.is_some() {
4866                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4867                    state.health.status = SupervisorHealthStatus::Failing;
4868                });
4869            }
4870            handle_health_probe_failure(
4871                spec,
4872                runtime,
4873                registry,
4874                process_liveness,
4875                snapshot,
4876                child,
4877                err,
4878                now_ms,
4879            )
4880            .await;
4881        }
4882    }
4883}
4884
4885pub(crate) struct HttpProbeTarget<'a> {
4886    address: std::net::SocketAddr,
4887    localhost: bool,
4888    authority: &'a str,
4889    path: String,
4890}
4891
4892/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4893/// or TLS. A URL cannot turn a local health check into an outbound connection.
4894pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4895    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4896        return Err("must not contain whitespace, controls, or a fragment".into());
4897    }
4898    let rest = url
4899        .strip_prefix("http://")
4900        .ok_or("must use plain http://")?;
4901    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4902    let (authority, suffix) = rest.split_at(split);
4903    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4904        ("::1", rest)
4905    } else {
4906        let split = authority.find(':').unwrap_or(authority.len());
4907        authority.split_at(split)
4908    };
4909    let ip: std::net::IpAddr = match host {
4910        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4911        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4912        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4913    };
4914    let port = if port.is_empty() {
4915        80
4916    } else {
4917        port.strip_prefix(':')
4918            .and_then(|p| p.parse::<u16>().ok())
4919            .filter(|p| *p > 0)
4920            .ok_or("must have a valid nonzero TCP port")?
4921    };
4922    let path = if suffix.is_empty() {
4923        "/".into()
4924    } else if suffix.starts_with('?') {
4925        format!("/{suffix}")
4926    } else {
4927        suffix.into()
4928    };
4929    Ok(HttpProbeTarget {
4930        address: std::net::SocketAddr::new(ip, port),
4931        localhost: host == "localhost",
4932        authority,
4933        path,
4934    })
4935}
4936
4937async fn probe_http_health(
4938    url: &str,
4939    deadline: Duration,
4940) -> Result<HealthReport, HealthProbeError> {
4941    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4942    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4943    // Keep partial diagnostics outside the timed future so cancellation does
4944    // not discard a status line or body bytes already received.
4945    let mut response_status = String::new();
4946    let mut body = Vec::new();
4947    let probe = async {
4948        // Resolve localhost ourselves so a hosts-file override cannot turn
4949        // this into an outbound request, while IPv6-only local servers work.
4950        let connection = match tokio::net::TcpStream::connect(target.address).await {
4951            Err(_) if target.localhost => {
4952                tokio::net::TcpStream::connect((
4953                    std::net::Ipv6Addr::LOCALHOST,
4954                    target.address.port(),
4955                ))
4956                .await
4957            }
4958            result => result,
4959        };
4960        let mut stream = connection.map_err(|error| {
4961            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4962        })?;
4963        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4964            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4965        let mut reader = BufReader::new(stream);
4966        let mut budget = 16 * 1024;
4967        let status = http_line(&mut reader, &mut budget).await?;
4968        let mut words = status.split_ascii_whitespace();
4969        let version = words.next();
4970        let code = words
4971            .next()
4972            .filter(|word| word.len() == 3)
4973            .and_then(|word| word.parse::<u16>().ok());
4974        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4975            || !code.is_some_and(|code| (100..600).contains(&code))
4976        {
4977            return Err(HealthProbeError::bad_answer(format!(
4978                "invalid HTTP status: {status}"
4979            )));
4980        }
4981        let code = code.expect("validated status code");
4982        response_status = status.clone();
4983        let mut length = None;
4984        let mut chunked = false;
4985        loop {
4986            let line = http_line(&mut reader, &mut budget).await?;
4987            if line.is_empty() {
4988                break;
4989            }
4990            if let Some((name, value)) = line.split_once(':') {
4991                if name.eq_ignore_ascii_case("content-length") {
4992                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4993                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4994                    })?);
4995                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4996                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4997                }
4998            }
4999        }
5000        if chunked {
5001            while body.len() < 200 {
5002                let line = http_line(&mut reader, &mut budget).await?;
5003                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5004                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5005                if size == 0 {
5006                    break;
5007                }
5008                let count = size.min((200 - body.len()) as u64) as usize;
5009                let start = body.len();
5010                (&mut reader)
5011                    .take(count as u64)
5012                    .read_to_end(&mut body)
5013                    .await
5014                    .map_err(|error| {
5015                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5016                    })?;
5017                if body.len() - start != count {
5018                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5019                }
5020                if size > count as u64 || body.len() == 200 {
5021                    break;
5022                }
5023                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5024                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5025                }
5026            }
5027        } else {
5028            reader
5029                .take(length.unwrap_or(200).min(200))
5030                .read_to_end(&mut body)
5031                .await
5032                .map_err(|error| {
5033                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5034                })?;
5035        }
5036        if (200..300).contains(&code) {
5037            Ok(HealthReport::ok())
5038        } else {
5039            Err(HealthProbeError::bad_answer(
5040                "HTTP health endpoint returned non-2xx",
5041            ))
5042        }
5043    };
5044    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5045        Err(HealthProbeError::no_answer(format!(
5046            "HTTP probe timed out after {deadline:?}"
5047        )))
5048    });
5049    if let Err(error) = &mut result {
5050        if !response_status.is_empty() {
5051            error.message = format!(
5052                "{}; {response_status}: {}",
5053                error.message,
5054                String::from_utf8_lossy(&body)
5055            );
5056        }
5057    }
5058    result
5059}
5060
5061async fn http_line(
5062    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5063    remaining: &mut usize,
5064) -> Result<String, HealthProbeError> {
5065    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5066    let mut line = Vec::new();
5067    (&mut *reader)
5068        .take(*remaining as u64)
5069        .read_until(b'\n', &mut line)
5070        .await
5071        .map_err(|error| {
5072            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5073        })?;
5074    *remaining -= line.len();
5075    if !line.ends_with(b"\r\n") {
5076        return Err(HealthProbeError::bad_answer(
5077            "HTTP headers are incomplete or exceed 16 KiB",
5078        ));
5079    }
5080    line.truncate(line.len() - 2);
5081    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5082}
5083
5084async fn probe_module_health(
5085    module_id: &str,
5086    runtime: &SupervisorRuntimeConfig,
5087    drain_deadline: Option<Instant>,
5088) -> Result<HealthReport, HealthProbeError> {
5089    let Some(forwarding) = runtime.forwarding.as_ref() else {
5090        return Err(HealthProbeError::misconfigured(
5091            "supervisor was not configured with a forwarding table",
5092        ));
5093    };
5094    let probe_started_at = Instant::now();
5095    let mut deadline = probe_started_at + runtime.health.deadline;
5096    if let Some(drain_deadline) = drain_deadline {
5097        deadline = deadline.min(drain_deadline);
5098    }
5099    let pending = if drain_deadline.is_some() {
5100        forwarding.begin_drain_health_probe_rpc_for(
5101            module_id,
5102            MODULE_CONTROL_OP_HEALTH_CHECK,
5103            probe_started_at,
5104            deadline,
5105        )
5106    } else {
5107        forwarding.begin_health_probe_rpc_for(
5108            module_id,
5109            MODULE_CONTROL_OP_HEALTH_CHECK,
5110            probe_started_at,
5111            deadline,
5112        )
5113    }
5114    .map_err(|err| {
5115        // The endpoint is not registered, so there is no live control lane to
5116        // ask. That is the module being absent, not slow.
5117        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5118    })?;
5119    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5120}
5121
5122/// [`probe_module_health`] for one endpoint rather than the id's active one.
5123///
5124/// A swap probes two processes that no by-id lookup reaches: its candidate
5125/// before cutover, and its superseded incumbent (for busy gauges) while the
5126/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5127/// bounds the by-id drain probe.
5128async fn probe_endpoint_health(
5129    endpoint: crate::ModuleEndpointId,
5130    runtime: &SupervisorRuntimeConfig,
5131    deadline_cap: Option<Instant>,
5132) -> Result<HealthReport, HealthProbeError> {
5133    let Some(forwarding) = runtime.forwarding.as_ref() else {
5134        return Err(HealthProbeError::misconfigured(
5135            "supervisor was not configured with a forwarding table",
5136        ));
5137    };
5138    let probe_started_at = Instant::now();
5139    let mut deadline = probe_started_at + runtime.health.deadline;
5140    if let Some(cap) = deadline_cap {
5141        deadline = deadline.min(cap);
5142    }
5143    let pending = forwarding
5144        .begin_endpoint_health_probe_rpc_for(
5145            endpoint,
5146            MODULE_CONTROL_OP_HEALTH_CHECK,
5147            probe_started_at,
5148            deadline,
5149        )
5150        .map_err(|err| {
5151            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5152        })?;
5153    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5154}
5155
5156/// Send a begun health probe and classify its answer.
5157async fn await_health_probe(
5158    forwarding: &ForwardingTable,
5159    pending: PendingModuleControlRpc,
5160    deadline: Instant,
5161    probe_budget: Duration,
5162) -> Result<HealthReport, HealthProbeError> {
5163    let PendingModuleControlRpc {
5164        endpoint,
5165        module_sink,
5166        negotiated_ver,
5167        corr,
5168        receiver,
5169    } = pending;
5170    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5171        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5172    })?;
5173    let frame = Frame::build_with_version(
5174        negotiated_ver,
5175        FrameType::Request,
5176        control_flags(),
5177        0,
5178        0,
5179        corr,
5180        body,
5181    )
5182    .map_err(|err| {
5183        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5184    })?;
5185
5186    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5187    // blocks waiting for capacity when the module's egress queue is full, and an
5188    // unbounded await here freezes the whole supervision actor (it stops polling
5189    // Child::wait and supervisor commands), making the module unrecoverable
5190    // in-band. On timeout the probe fails like any transport failure.
5191    match timeout_at(deadline, module_sink.send(frame)).await {
5192        Ok(Ok(())) => {}
5193        Ok(Err(err)) => {
5194            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5195            // A closed sink means the module's egress channel is gone -- the
5196            // receiving half is dropped when its connection tears down. Proof.
5197            return Err(HealthProbeError::lane_dead(format!(
5198                "failed to send health.check: {err}"
5199            )));
5200        }
5201        Err(_elapsed) => {
5202            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5203            // A full egress queue means the module is not draining its socket, which
5204            // is consistent with a wedged module AND with one whose reader is merely
5205            // starved. Silence, not proof.
5206            return Err(HealthProbeError::no_answer(
5207                "health.check send timed out before enqueue (module egress full)",
5208            ));
5209        }
5210    }
5211
5212    match timeout_at(deadline, receiver).await {
5213        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5214        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5215        // and those prove it is alive even though the probe failed.
5216        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5217            response.health_report().ok_or_else(|| {
5218                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5219            })
5220        }
5221        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5222            format!("health.check rejected: {}", body.message),
5223        )),
5224        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5225            Err(HealthProbeError::lane_dead(message))
5226        }
5227        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5228            Err(HealthProbeError::bad_answer(message))
5229        }
5230        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5231            Err(HealthProbeError::bad_answer(format!(
5232                "expected module-control op '{expected}', got '{actual}'"
5233            )))
5234        }
5235        // A reply that crosses the deadline before this waiter observes it is
5236        // still proof of life. The forwarding path records its end-to-end latency
5237        // before delivering this classification.
5238        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5239            "module answered health.check after its daemon deadline",
5240        )),
5241        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5242            "health.check waiter was canceled before the module responded",
5243        )),
5244        Err(_) => {
5245            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5246            Err(HealthProbeError::no_answer(format!(
5247                "module did not answer health.check within {probe_budget:?}"
5248            )))
5249        }
5250    }
5251}
5252
5253#[allow(clippy::too_many_arguments)]
5254async fn handle_health_report(
5255    spec: &ModuleSpec,
5256    runtime: &SupervisorRuntimeConfig,
5257    registry: &Registry,
5258    process_liveness: &SupervisorProcessLiveness,
5259    snapshot: &SharedSnapshot,
5260    child: &mut Option<SupervisedChild>,
5261    report: HealthReport,
5262    now_ms: u64,
5263) {
5264    let status = supervisor_health_status(report.status);
5265    let detail = report.detail.clone();
5266    let metrics = truncate_health_metrics(report.metrics);
5267    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5268        state.health.status = status;
5269        state.health.last_probe_ms = Some(now_ms);
5270        state.health.detail = detail.clone();
5271        state.health.metrics = metrics.clone();
5272        state.health.consecutive_failures = 0;
5273        state.failed_retry.observe_health(
5274            report.status == HealthStatus::Ok,
5275            runtime.restart_policy.window,
5276        );
5277    });
5278
5279    let action = match report.status {
5280        HealthStatus::Ok => return,
5281        HealthStatus::Degraded => runtime.health.on_degraded,
5282        HealthStatus::Failing => runtime.health.on_failing,
5283    };
5284    apply_l3_health_action(
5285        spec,
5286        runtime,
5287        registry,
5288        process_liveness,
5289        snapshot,
5290        child,
5291        status,
5292        detail.as_deref(),
5293        action,
5294        now_ms,
5295    )
5296    .await;
5297}
5298
5299#[allow(clippy::too_many_arguments)]
5300async fn handle_health_probe_failure(
5301    spec: &ModuleSpec,
5302    runtime: &SupervisorRuntimeConfig,
5303    registry: &Registry,
5304    process_liveness: &SupervisorProcessLiveness,
5305    snapshot: &SharedSnapshot,
5306    child: &mut Option<SupervisedChild>,
5307    err: HealthProbeError,
5308    now_ms: u64,
5309) {
5310    let threshold = runtime.health.failure_threshold.max(1);
5311    let mut failures = 0;
5312    // Carry the evidence class into the operator-visible detail. Without it,
5313    // "module did not answer within 5s" and "the control lane is gone" are two
5314    // prose strings in the same field, and the reader has to know the codebase to
5315    // tell which one is proof of anything.
5316    let detail = format!("[{}] {err}", err.label());
5317    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5318        state
5319            .failed_retry
5320            .observe_health(false, runtime.restart_policy.window);
5321        // A failed wire probe invalidates the last report, even before the
5322        // restart threshold. HTTP probes already mark failures as Failing.
5323        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5324            state.health.status = SupervisorHealthStatus::Unknown;
5325        }
5326        state.health.last_probe_ms = Some(now_ms);
5327        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5328        state.health.detail = Some(detail.clone());
5329        state.health.metrics = None;
5330        failures = state.health.consecutive_failures;
5331    });
5332
5333    if failures < threshold {
5334        warn!(
5335            module_id = %spec.module_id,
5336            consecutive_failures = failures,
5337            threshold,
5338            evidence = err.label(),
5339            detail = %detail,
5340            "health.check probe failed"
5341        );
5342        return;
5343    }
5344
5345    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5346        state.state = ModuleState::Unresponsive;
5347        state.health.status = SupervisorHealthStatus::Unresponsive;
5348    });
5349    // The evidence class is logged at the kill site because this is the line an
5350    // operator reads after an unexplained restart. A streak of `no-answer` under
5351    // machine load is the known false-positive shape; a `lane-dead` is not.
5352    if runtime.health.critical {
5353        error!(
5354            module_id = %spec.module_id,
5355            status = "unresponsive",
5356            evidence = err.label(),
5357            detail = %detail,
5358            "critical module health alert"
5359        );
5360    } else {
5361        warn!(
5362            module_id = %spec.module_id,
5363            status = "unresponsive",
5364            evidence = err.label(),
5365            detail = %detail,
5366            "module health threshold breached"
5367        );
5368    }
5369    if let Err(err) = health_restart_child(
5370        spec,
5371        runtime,
5372        registry,
5373        process_liveness,
5374        snapshot,
5375        child,
5376        SupervisorHealthStatus::Unresponsive,
5377        Some(&detail),
5378        now_ms,
5379    )
5380    .await
5381    {
5382        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5383    }
5384}
5385
5386#[allow(clippy::too_many_arguments)]
5387async fn apply_l3_health_action(
5388    spec: &ModuleSpec,
5389    runtime: &SupervisorRuntimeConfig,
5390    registry: &Registry,
5391    process_liveness: &SupervisorProcessLiveness,
5392    snapshot: &SharedSnapshot,
5393    child: &mut Option<SupervisedChild>,
5394    status: SupervisorHealthStatus,
5395    detail: Option<&str>,
5396    action: HealthAction,
5397    now_ms: u64,
5398) {
5399    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5400    match action {
5401        HealthAction::Report => {
5402            info!(
5403                module_id = %spec.module_id,
5404                status = ?status,
5405                detail,
5406                "module reported non-ok health"
5407            );
5408        }
5409        HealthAction::Alert => {
5410            error!(
5411                module_id = %spec.module_id,
5412                status = ?status,
5413                detail,
5414                "module health alert"
5415            );
5416        }
5417        HealthAction::Restart => {
5418            if let Err(err) = health_restart_child(
5419                spec,
5420                runtime,
5421                registry,
5422                process_liveness,
5423                snapshot,
5424                child,
5425                status,
5426                detail,
5427                now_ms,
5428            )
5429            .await
5430            {
5431                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5432            }
5433        }
5434    }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn health_restart_child(
5439    spec: &ModuleSpec,
5440    runtime: &SupervisorRuntimeConfig,
5441    registry: &Registry,
5442    process_liveness: &SupervisorProcessLiveness,
5443    snapshot: &SharedSnapshot,
5444    child: &mut Option<SupervisedChild>,
5445    status: SupervisorHealthStatus,
5446    detail: Option<&str>,
5447    now_ms: u64,
5448) -> Result<(), SuperviseError> {
5449    let (enabled, schedule) = {
5450        let mut state = lock_snapshot(snapshot)?;
5451        let enabled = state.enabled;
5452        let schedule = if enabled {
5453            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5454        } else {
5455            None
5456        };
5457        (enabled, schedule)
5458    };
5459
5460    if !enabled {
5461        return Err(SuperviseError::Disabled {
5462            module_id: spec.module_id.clone(),
5463        });
5464    }
5465
5466    if schedule.is_none() {
5467        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5468        error!(
5469            module_id = %spec.module_id,
5470            status = ?status,
5471            detail,
5472            max_restarts = runtime.restart_policy.max_restarts,
5473            window_secs = runtime.restart_policy.window.as_secs(),
5474            reason = %runtime.restart_policy.budget_exhausted_detail(),
5475            "health restart budget exhausted; marking module failed"
5476        );
5477        let stop_notice = begin_forwarding_drain_if_configured(
5478            spec,
5479            runtime,
5480            registry,
5481            snapshot,
5482            Some(true),
5483            RouteCloseReason::Disable,
5484        )
5485        .await?;
5486        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5487            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5488        })?;
5489        let result = drain_optional_child(
5490            &spec.module_id,
5491            spec.protocol,
5492            stop_notice,
5493            registry,
5494            runtime.forwarding.as_deref(),
5495            snapshot,
5496            &runtime.terminal_ring,
5497            &runtime.spawn_events,
5498            child,
5499            runtime.drain_timeout,
5500            ModuleState::Failed,
5501            Some(true),
5502            &runtime.failure_context(true),
5503        )
5504        .await;
5505        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5507                state.failed_retry.schedule(&spec.module_id);
5508            }
5509        })?;
5510        result?;
5511        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5512        return Ok(());
5513    }
5514
5515    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5516    let mut restart_count = 0;
5517    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5518        restart_count = state.crash_restarts.len();
5519        state.state = ModuleState::Unresponsive;
5520        state.health.status = status;
5521        state.health.last_action = Some(HealthAction::Restart.to_string());
5522        state.health.last_action_ms = Some(now_ms);
5523    })?;
5524    warn!(
5525        module_id = %spec.module_id,
5526        status = ?status,
5527        detail,
5528        restart_count,
5529        restart_in_window = schedule.restart_in_window,
5530        delay_ms = schedule.delay.as_millis() as u64,
5531        "health-triggered module restart"
5532    );
5533
5534    let stop_notice = begin_forwarding_drain_if_configured(
5535        spec,
5536        runtime,
5537        registry,
5538        snapshot,
5539        Some(true),
5540        RouteCloseReason::Restart,
5541    )
5542    .await?;
5543    drain_optional_child(
5544        &spec.module_id,
5545        spec.protocol,
5546        stop_notice,
5547        registry,
5548        runtime.forwarding.as_deref(),
5549        snapshot,
5550        &runtime.terminal_ring,
5551        &runtime.spawn_events,
5552        child,
5553        runtime.drain_timeout,
5554        ModuleState::Restarting,
5555        Some(true),
5556        &runtime.failure_context(true),
5557    )
5558    .await?;
5559    schedule_respawn(
5560        runtime,
5561        snapshot,
5562        &spec.module_id,
5563        schedule.delay,
5564        RespawnKind::Spawn,
5565        true,
5566    )
5567}
5568
5569fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5570    if let Some(reply) = runtime
5571        .deferred_reload_reply
5572        .lock()
5573        .unwrap_or_else(|p| p.into_inner())
5574        .take()
5575    {
5576        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5577            module_id: module_id.to_string(),
5578            reason: reason.to_string(),
5579        }));
5580    }
5581}
5582
5583fn schedule_respawn(
5584    runtime: &SupervisorRuntimeConfig,
5585    snapshot: &SharedSnapshot,
5586    module_id: &str,
5587    delay: Duration,
5588    kind: RespawnKind,
5589    daemon_initiated: bool,
5590) -> Result<(), SuperviseError> {
5591    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5592    update_snapshot(snapshot, Some(module_id), |state| {
5593        state.respawn_pending = true
5594    })?;
5595    *runtime
5596        .scheduled_respawn
5597        .lock()
5598        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5599        deadline: Instant::now() + delay,
5600        kind,
5601        daemon_initiated,
5602    });
5603    Ok(())
5604}
5605
5606fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5607    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5608        state.health.last_action = Some(action);
5609        state.health.last_action_ms = Some(now_ms);
5610    });
5611}
5612
5613fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5614    match status {
5615        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5616        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5617        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5618    }
5619}
5620
5621/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5622/// returned to every `supervisor.list` and `supervisor.health` caller.
5623///
5624/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5625/// path: that request exists to return a module's complete metrics object, and
5626/// `ck health <module-id>` documents it as the way to see what the cached view
5627/// truncates. The asymmetry is the feature.
5628///
5629/// So a new caller must decide which side it is on rather than assume the cap is
5630/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5631/// the truncation that path exists to avoid.
5632fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5633    let metrics = metrics?;
5634    match serde_json::to_vec(&metrics) {
5635        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5636            "truncated": true,
5637            "original_bytes": encoded.len(),
5638        })),
5639        Ok(_) | Err(_) => Some(metrics),
5640    }
5641}
5642
5643/// Spread health probes so a fleet-wide restart does not converge them.
5644///
5645/// The delay is derived from the module id and probe index rather than a random
5646/// source, so it is deterministic per module: a module keeps its own offset
5647/// across daemon restarts instead of re-rolling into a collision.
5648fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5649    if cadence.is_zero() {
5650        return Duration::ZERO;
5651    }
5652    let cadence_ms = cadence.as_millis() as u64;
5653    // This early return is REDUNDANT, deliberately, and a mutation run will show
5654    // it surviving removal. Recording why here so the next person to notice does
5655    // not have to re-derive it:
5656    //
5657    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5658    //   a zero cadence and builds the Duration from whole milliseconds, so a
5659    //   sub-millisecond cadence cannot come from config.
5660    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5661    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5662    //   -- exactly what this returns.
5663    //
5664    // Kept as a guard against a future widening of the config parser (accepting
5665    // microseconds, say), which would make the sub-millisecond case reachable.
5666    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5667    // divides by zero. Remove this and nothing changes.
5668    if cadence_ms == 0 {
5669        return cadence;
5670    }
5671    // Note that this never returns less than one cadence, including for the FIRST
5672    // probe. So a freshly registered module reports health `unknown` for a full
5673    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5674    // ready to answer.
5675    //
5676    // That is a property of the supervisor's schedule, not of any module: an
5677    // operator watching a restart sees `unknown` and cannot tell it from a module
5678    // that is slow to warm. Measured on two unrelated modules, both flipping to
5679    // `ok` between 22s and 32s after restart.
5680    //
5681    // Left as-is because spreading the first probe is what keeps a fleet-wide
5682    // restart from firing fourteen simultaneous probes into a cold machine. The
5683    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5684    // that thundering herd for a faster first reading.
5685    let jitter_span = (cadence_ms / 10).max(1);
5686    let hash = module_id.as_bytes().iter().fold(
5687        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5688        |acc, byte| {
5689            acc.wrapping_mul(1099511628211)
5690                .wrapping_add(u64::from(*byte))
5691        },
5692    );
5693    cadence + Duration::from_millis(hash % jitter_span)
5694}
5695
5696#[cfg(test)]
5697mod tests {
5698    use super::*;
5699
5700    #[test]
5701    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5702        let handle = SupervisorHandle::new();
5703        let module_id = "readded-tombstone";
5704        handle.record_rescan_removal(module_id);
5705        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5706
5707        handle.apply_identity_configuration(&ModuleSpec {
5708            module_id: module_id.to_string(),
5709            program: PathBuf::from("/test/module"),
5710            args: Vec::new(),
5711            env: Vec::new(),
5712            reserved: false,
5713            reserved_prefixes: Vec::new(),
5714            protocol: ModuleProtocol::Subc,
5715            overlap: Default::default(),
5716        });
5717
5718        assert!(
5719            handle.removal_tombstone_age_ms(module_id).is_none(),
5720            "a re-added module must not retain a stale removal tombstone"
5721        );
5722    }
5723
5724    /// What one module's owner looked like from the control plane at the
5725    /// instant after its first process was spawned.
5726    #[derive(Debug, PartialEq, Eq)]
5727    struct OwnerInSpawnWindow {
5728        module_id: String,
5729        configured: bool,
5730        on_roster: bool,
5731        admission_refusal: Option<&'static str>,
5732    }
5733
5734    /// A supervised module's process can connect, register, sync its scopes
5735    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5736    /// the module on the roster until afterward; configured supervision does
5737    /// so before spawning. In both cases the owner must
5738    /// already read as configured, so a scoped `route.open` against it is
5739    /// refused as retryable `scope_not_synced` and not as terminal
5740    /// `scope_not_live` ("will never sync").
5741    ///
5742    /// The hook runs just after each initial spawn attempt, before returning to
5743    /// the caller, so no race with a real child's registration is needed.
5744    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5745    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5746        use crate::scopes::ScopeTable;
5747        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5748
5749        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5750        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5751            module_id: module_id.to_string(),
5752            program,
5753            args: Vec::new(),
5754            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5755                .into_iter()
5756                .map(|key| (key.to_string(), dir.path().display().to_string()))
5757                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5758                .collect(),
5759            reserved: false,
5760            reserved_prefixes: Vec::new(),
5761            protocol: ModuleProtocol::Subc,
5762            overlap: Default::default(),
5763        };
5764        let live = super::terminal_history_tests::fake_aft_stub_path();
5765        let missing = dir.path().join("definitely-missing-module");
5766
5767        let handle = SupervisorHandle::new();
5768        let mut supervisor =
5769            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5770                .with_handle(handle.clone());
5771        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5772        let hook_handle = handle.clone();
5773        let hook_observed = Arc::clone(&observed);
5774        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5775            // Exactly what the control plane computes for a scoped route.open
5776            // naming this module as the owner of a scope it has not synced.
5777            let configured = hook_handle.is_configured(module_id);
5778            let selector = ScopeSelector::new(
5779                Principal::Reserved {
5780                    module_id: module_id.to_string(),
5781                },
5782                "s".to_string(),
5783                Some(1),
5784            );
5785            let carrier = Principal::Reserved {
5786                module_id: "carrier".to_string(),
5787            };
5788            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5789                .admit(&carrier, module_id, &selector, configured)
5790            {
5791                Ok(_) => None,
5792                Err(refusal) => Some(refusal.code),
5793            };
5794            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5795                module_id: module_id.to_string(),
5796                configured,
5797                on_roster: hook_handle.get(module_id).is_some(),
5798                admission_refusal,
5799            });
5800        })));
5801
5802        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5803        let configured = supervisor
5804            .supervise_configured(stub("configured", live.clone()), true)
5805            .unwrap();
5806        let with_health = supervisor
5807            .supervise_configured_with_health(
5808                stub("with-health", live.clone()),
5809                true,
5810                HealthConfig::default(),
5811                None,
5812                RestartPolicy::default(),
5813            )
5814            .unwrap();
5815        // The failed-spawn path still puts the module on the roster (as
5816        // failed), so it is configured throughout.
5817        let failed = supervisor
5818            .supervise_configured_with_health(
5819                stub("failed-spawn", missing.clone()),
5820                true,
5821                HealthConfig::default(),
5822                None,
5823                RestartPolicy::default(),
5824            )
5825            .unwrap();
5826        // A failed plain `spawn` puts nothing on the roster, so its mark is
5827        // taken back once the spawn has failed.
5828        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5829
5830        let expected = [
5831            "plain",
5832            "configured",
5833            "with-health",
5834            "failed-spawn",
5835            "spawn-error",
5836        ]
5837        .into_iter()
5838        .map(|module_id| OwnerInSpawnWindow {
5839            module_id: module_id.to_string(),
5840            configured: true,
5841            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5842            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5843        })
5844        .collect::<Vec<_>>();
5845        assert_eq!(*observed.lock().unwrap(), expected);
5846
5847        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5848            assert!(
5849                handle.get(module_id).is_some(),
5850                "{module_id} is on the roster"
5851            );
5852            assert!(
5853                handle.is_configured(module_id),
5854                "{module_id} stays configured"
5855            );
5856        }
5857        assert!(handle.get("spawn-error").is_none());
5858        assert!(
5859            !handle.is_configured("spawn-error"),
5860            "a plain spawn that failed must not leave its module marked configured"
5861        );
5862
5863        // Leaving the roster clears the mark with it.
5864        handle.retire("failed-spawn");
5865        assert!(!handle.is_configured("failed-spawn"));
5866
5867        for module in [plain, configured, with_health] {
5868            module.stop().await.unwrap();
5869        }
5870        drop(failed);
5871    }
5872
5873    /// A configured module reads as starting until the supervisor has admitted
5874    /// its first process (exec confirmed, identity durably recorded), even
5875    /// though that process is already running by then and can register over
5876    /// HELLO, which is what lists it in catalog.list. So a client that needs
5877    /// the supervisor state must wait for that state; seeing the module in
5878    /// the catalog does not mean supervisor.list already says running.
5879    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5880    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5881        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5882        let spec = ModuleSpec {
5883            module_id: "spawned-not-admitted".to_string(),
5884            program: super::terminal_history_tests::fake_aft_stub_path(),
5885            args: Vec::new(),
5886            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5887                .into_iter()
5888                .map(|key| (key.to_string(), dir.path().display().to_string()))
5889                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5890                .collect(),
5891            reserved: false,
5892            reserved_prefixes: Vec::new(),
5893            protocol: ModuleProtocol::Subc,
5894            overlap: Default::default(),
5895        };
5896        let handle = SupervisorHandle::new();
5897        let mut supervisor =
5898            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5899                .with_handle(handle.clone());
5900        let observed = Arc::new(Mutex::new(None));
5901        let hook_handle = handle.clone();
5902        let hook_observed = Arc::clone(&observed);
5903        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5904            let state = hook_handle
5905                .get(module_id)
5906                .expect("a configured module is on the roster before it is spawned")
5907                .state()
5908                .unwrap();
5909            *hook_observed.lock().unwrap() = Some(state);
5910        })));
5911
5912        let module = supervisor.supervise_configured(spec, true).unwrap();
5913        assert_eq!(
5914            *observed.lock().unwrap(),
5915            Some(ModuleState::Starting),
5916            "between the spawn and its admission the supervisor still reports starting"
5917        );
5918        assert_eq!(module.state().unwrap(), ModuleState::Running);
5919        module.stop().await.unwrap();
5920    }
5921
5922    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5923        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5924        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5925            snapshot.process_alive = true;
5926            snapshot.pid = Some(41);
5927            snapshot.spawned_at_ms = Some(42);
5928            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5929            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5930            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5931                device: 43,
5932                inode: 44,
5933                #[cfg(windows)]
5934                windows_image: None,
5935            });
5936        })
5937        .unwrap();
5938        snapshot
5939    }
5940
5941    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5942        let snapshot = lock_snapshot(snapshot).unwrap();
5943        assert!(!snapshot.process_alive);
5944        assert_eq!(snapshot.pid, None);
5945        assert_eq!(snapshot.spawned_at_ms, None);
5946        assert_eq!(snapshot.spawned_from, None);
5947        assert_eq!(snapshot.launch_spec_digest, None);
5948        assert_eq!(snapshot.spawned_file_identity, None);
5949    }
5950
5951    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5952    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5953        let supervisor =
5954            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5955        let mut runtime = supervisor.runtime_config();
5956        runtime.test_seed_stale_facts_before_enable_spawn = true;
5957        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5958        let mut child = None;
5959        let spec = ModuleSpec {
5960            module_id: "failed-enable-clears-facts".to_string(),
5961            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5962            args: Vec::new(),
5963            env: Vec::new(),
5964            reserved: false,
5965            reserved_prefixes: Vec::new(),
5966            protocol: ModuleProtocol::Subc,
5967            overlap: Default::default(),
5968        };
5969
5970        let result = set_child_enabled(
5971            &spec,
5972            &runtime,
5973            &supervisor.registry,
5974            &supervisor.process_liveness,
5975            &snapshot,
5976            &mut child,
5977            true,
5978            &runtime.failure_context(false),
5979        )
5980        .await;
5981
5982        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5983        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5984        assert_snapshot_process_facts_cleared(&snapshot);
5985    }
5986
5987    #[tokio::test]
5988    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5989        let supervisor =
5990            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5991        let runtime = supervisor.runtime_config();
5992        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5993            ModuleState::Restarting,
5994            true,
5995        )));
5996        let spec = ModuleSpec {
5997            module_id: "start-stranded-restarting".to_string(),
5998            program: super::terminal_history_tests::fake_aft_stub_path(),
5999            args: Vec::new(),
6000            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6001            reserved: false,
6002            reserved_prefixes: Vec::new(),
6003            protocol: ModuleProtocol::None,
6004            overlap: Default::default(),
6005        };
6006        let mut child = None;
6007        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6008        assert!(!super::set_child_enabled(
6009            &spec,
6010            &runtime,
6011            &Registry::default(),
6012            &supervisor.process_liveness,
6013            &snapshot,
6014            &mut child,
6015            true,
6016            &runtime.failure_context(false),
6017        )
6018        .await
6019        .unwrap());
6020        assert!(child.is_none());
6021        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6022        assert!(super::set_child_enabled(
6023            &spec,
6024            &runtime,
6025            &Registry::default(),
6026            &supervisor.process_liveness,
6027            &snapshot,
6028            &mut child,
6029            true,
6030            &runtime.failure_context(false),
6031        )
6032        .await
6033        .unwrap());
6034        assert_eq!(
6035            lock_snapshot(&snapshot).unwrap().state,
6036            ModuleState::Running
6037        );
6038        let mut child = child.unwrap();
6039        child.start_kill().unwrap();
6040        child.wait().await.unwrap();
6041    }
6042
6043    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6044    async fn failed_reload_spawn_clears_current_process_facts() {
6045        let supervisor =
6046            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6047        let mut runtime = supervisor.runtime_config();
6048        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6049        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6050        let mut child = None;
6051        let spec = ModuleSpec {
6052            module_id: "failed-reload-clears-facts".to_string(),
6053            program: PathBuf::from("/unused/failed-reload-module"),
6054            args: Vec::new(),
6055            env: Vec::new(),
6056            reserved: false,
6057            reserved_prefixes: Vec::new(),
6058            protocol: ModuleProtocol::Subc,
6059            overlap: Default::default(),
6060        };
6061
6062        let result = handle_reload_spawn_failure(
6063            &spec,
6064            &runtime,
6065            &supervisor.process_liveness,
6066            &snapshot,
6067            &mut child,
6068            "forced reload spawn failure".to_string(),
6069        )
6070        .await;
6071
6072        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6073        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6074        assert_snapshot_process_facts_cleared(&snapshot);
6075    }
6076
6077    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6078    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6079        let supervisor =
6080            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6081        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6082        let module = supervisor.supervised_module(
6083            ModuleSpec {
6084                module_id: "drop-clears-facts".to_string(),
6085                program: PathBuf::from("/unused/drop-module"),
6086                args: Vec::new(),
6087                env: Vec::new(),
6088                reserved: false,
6089                reserved_prefixes: Vec::new(),
6090                protocol: ModuleProtocol::Subc,
6091                overlap: Default::default(),
6092            },
6093            supervisor.runtime_config(),
6094            Arc::clone(&snapshot),
6095            None,
6096        );
6097        assert!(!module
6098            .inner
6099            .monitor
6100            .lock()
6101            .unwrap()
6102            .as_ref()
6103            .unwrap()
6104            .is_finished());
6105
6106        drop(module);
6107
6108        assert_eq!(
6109            lock_snapshot(&snapshot).unwrap().state,
6110            ModuleState::Stopped
6111        );
6112        assert_snapshot_process_facts_cleared(&snapshot);
6113    }
6114
6115    #[cfg(unix)]
6116    #[tokio::test]
6117    async fn rescan_preserves_running_protocol_until_respawn() {
6118        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6119        let initial = ModuleSpec {
6120            module_id: "rescan-protocol".into(),
6121            program: PathBuf::from("/bin/sleep"),
6122            args: vec!["60".into()],
6123            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6124                .into_iter()
6125                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6126                .collect(),
6127            reserved: false,
6128            reserved_prefixes: vec![],
6129            protocol: ModuleProtocol::None,
6130            overlap: Default::default(),
6131        };
6132        let supervisor =
6133            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6134        let module = supervisor.spawn(initial.clone()).unwrap();
6135        assert!(module.status().unwrap().live);
6136        let mut next = initial;
6137        next.protocol = ModuleProtocol::Subc;
6138        module
6139            .update_configuration(next.clone(), HealthConfig::default(), None)
6140            .await
6141            .unwrap();
6142        assert!(
6143            module.status().unwrap().live,
6144            "rescan must not require HELLO from the old non-wire process"
6145        );
6146        let runtime = supervisor.runtime_config();
6147        let action = on_child_exit(
6148            &next,
6149            RestartPolicy::default(),
6150            &supervisor.registry,
6151            &module.inner.snapshot,
6152            &runtime.terminal_ring,
6153            &runtime.spawn_events,
6154            &runtime.child_roster,
6155            ExitReport {
6156                kind: ExitKind::Clean,
6157                code: Some(0),
6158                signal: None,
6159                at_ms: unix_ms_now(),
6160            },
6161            true,
6162        )
6163        .await;
6164        assert!(
6165            matches!(action, NextAction::Restart { .. }),
6166            "the old non-wire process's clean exit must restart"
6167        );
6168        module.drain().await.unwrap();
6169    }
6170
6171    #[cfg(unix)]
6172    #[tokio::test]
6173    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6174        use std::os::unix::fs::PermissionsExt;
6175        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6176        let script = dir.join("module.sh");
6177        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6178        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6179        let record_path = dir.join("live-children.json");
6180        let supervisor =
6181            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6182                .with_live_children_record(&record_path);
6183        for (program, args) in [
6184            (PathBuf::from("sleep"), vec!["60".into()]),
6185            (script, vec![]),
6186        ] {
6187            let spec = ModuleSpec {
6188                module_id: "image-identity".into(),
6189                program: program.clone(),
6190                args,
6191                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6192                    .into_iter()
6193                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6194                    .collect(),
6195                reserved: false,
6196                reserved_prefixes: vec![],
6197                protocol: ModuleProtocol::None,
6198                overlap: Default::default(),
6199            };
6200            let module = supervisor.spawn(spec).unwrap();
6201            #[cfg(target_os = "macos")]
6202            {
6203                // SETEXEC confirmation is asynchronous; the orphan record must
6204                // identify the final image, never the intermediate trampoline.
6205                let deadline = Instant::now() + Duration::from_secs(5);
6206                while crate::live_children::read_record(&record_path)
6207                    .unwrap()
6208                    .iter()
6209                    .all(|entry| entry.executable.is_none())
6210                {
6211                    assert!(Instant::now() < deadline, "module image was not confirmed");
6212                    tokio::time::sleep(Duration::from_millis(5)).await;
6213                }
6214            }
6215            let entry = crate::live_children::read_record(&record_path)
6216                .unwrap()
6217                .pop()
6218                .unwrap();
6219            let observed = subc_os::Process::open(entry.pid)
6220                .unwrap()
6221                .unwrap()
6222                .observe()
6223                .unwrap();
6224            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6225            module.drain().await.unwrap();
6226            assert_eq!(
6227                verdict,
6228                crate::live_children::IdentityVerdict::Matches,
6229                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6230            );
6231        }
6232    }
6233
6234    #[cfg(unix)]
6235    fn http_fixture(
6236        dir: &std::path::Path,
6237        url: &str,
6238        threshold: u32,
6239    ) -> crate::daemon_config::ConfiguredModule {
6240        let path = dir.join("subc.jsonc");
6241        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6242            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6243            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6244            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6245            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6246        }}}).to_string()).unwrap();
6247        crate::daemon_config::load(&path)
6248            .unwrap()
6249            .unwrap()
6250            .modules
6251            .pop()
6252            .unwrap()
6253    }
6254
6255    #[cfg(unix)]
6256    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6257        timeout(Duration::from_secs(5), async {
6258            loop {
6259                if module.status().unwrap().health.status == status {
6260                    break;
6261                }
6262                sleep(Duration::from_millis(5)).await;
6263            }
6264        })
6265        .await
6266        .unwrap_or_else(|_| {
6267            panic!(
6268                "expected {status:?}, got {:?}",
6269                module.status().unwrap().health
6270            )
6271        });
6272    }
6273
6274    #[cfg(unix)]
6275    #[tokio::test]
6276    async fn http_health_status_flips_ok_failing_ok() {
6277        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6278        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6279        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6280        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6281        let serving_status = status.clone();
6282        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6283        let server = tokio::spawn(async move {
6284            loop {
6285                let (mut stream, _) = listener.accept().await.unwrap();
6286                let mut request = [0u8; 2048];
6287                let count = stream.read(&mut request).await.unwrap();
6288                assert!(count > 0, "a probe must send an HTTP request");
6289                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6290                let body = if code == 200 {
6291                    "ready"
6292                } else {
6293                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6294                };
6295                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6296                let _ = stream.write_all(response.as_bytes()).await;
6297            }
6298        });
6299        let configured = http_fixture(&dir, &url, 1000);
6300        let module =
6301            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6302                .supervise_configured_with_health(
6303                    configured.module_spec(),
6304                    true,
6305                    configured.health,
6306                    configured.drain_timeout_ms,
6307                    configured.restart,
6308                )
6309                .unwrap();
6310        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6311        status.store(503, std::sync::atomic::Ordering::SeqCst);
6312        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6313        assert!(module
6314            .status()
6315            .unwrap()
6316            .health
6317            .detail
6318            .unwrap()
6319            .contains("scratch failure"));
6320        status.store(200, std::sync::atomic::Ordering::SeqCst);
6321        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6322        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6323        let before = module.status().unwrap();
6324        let (spec, mut health) = module.configuration().unwrap();
6325        health.http = None;
6326        module
6327            .update_configuration(spec.clone(), health.clone(), Some(10))
6328            .await
6329            .unwrap();
6330        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6331        health.http = Some(url);
6332        module
6333            .update_configuration(spec, health, Some(10))
6334            .await
6335            .unwrap();
6336        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6337        assert_eq!(
6338            module.status().unwrap().pid,
6339            before.pid,
6340            "changing a probe must apply live, not restart its process"
6341        );
6342        let (spec, mut health) = module.configuration().unwrap();
6343        health.failure_threshold = 2;
6344        module
6345            .update_configuration(spec, health, Some(10))
6346            .await
6347            .unwrap();
6348        status.store(503, std::sync::atomic::Ordering::SeqCst);
6349        timeout(Duration::from_secs(5), async {
6350            while module.status().unwrap().spawn_generation == before.spawn_generation {
6351                sleep(Duration::from_millis(5)).await;
6352            }
6353        })
6354        .await
6355        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6356        module.drain().await.unwrap();
6357        server.abort();
6358    }
6359
6360    #[cfg(unix)]
6361    #[tokio::test]
6362    async fn http_health_replacement_recovers_after_one_restart() {
6363        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6364        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6365        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6366        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6367        let mut configured = http_fixture(&dir, &url, 2);
6368        configured.restart.max_restarts = 2;
6369        let module =
6370            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6371                .supervise_configured_with_health(
6372                    configured.module_spec(),
6373                    true,
6374                    configured.health,
6375                    configured.drain_timeout_ms,
6376                    configured.restart,
6377                )
6378                .unwrap();
6379        let first_generation = module.status().unwrap().spawn_generation;
6380        let observed = module.clone();
6381        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6382        let server = tokio::spawn(async move {
6383            let mut generation = first_generation;
6384            let mut probes = 0;
6385            loop {
6386                let (mut stream, _) = listener.accept().await.unwrap();
6387                let mut request = [0; 2048];
6388                assert!(stream.read(&mut request).await.unwrap() > 0);
6389                let current = observed.status().unwrap().spawn_generation;
6390                if current != generation {
6391                    generation = current;
6392                    probes = 0;
6393                }
6394                probes += 1;
6395                // The original process never recovers. Each replacement has one
6396                // transient miss, then answers successfully for its remaining life.
6397                let healthy = generation != first_generation && probes > 1;
6398                let code = if healthy { 200 } else { 503 };
6399                let response = format!(
6400                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6401                );
6402                let _ = stream.write_all(response.as_bytes()).await;
6403                if healthy {
6404                    healthy_tx.send(()).await.unwrap();
6405                }
6406            }
6407        });
6408        timeout(Duration::from_secs(5), async {
6409            let mut healthy_probes = 0;
6410            loop {
6411                if module.state().unwrap() == ModuleState::Failed {
6412                    break;
6413                }
6414                while healthy_rx.try_recv().is_ok() {
6415                    healthy_probes += 1;
6416                }
6417                if healthy_probes >= 3
6418                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6419                {
6420                    break;
6421                }
6422                sleep(Duration::from_millis(5)).await;
6423            }
6424        })
6425        .await
6426        .expect("the replacement must settle healthy without exhausting its restart budget");
6427        let status = module.status().unwrap();
6428        module.drain().await.unwrap();
6429        server.abort();
6430        assert_eq!(status.state, ModuleState::Running);
6431        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6432        assert_eq!(status.health.consecutive_failures, 0);
6433        assert_eq!(status.lifetime_restarts, 1);
6434        assert_eq!(status.restart_count, 1);
6435    }
6436
6437    #[cfg(unix)]
6438    #[tokio::test]
6439    async fn http_health_sustained_failure_exhausts_restart_budget() {
6440        use std::collections::BTreeMap;
6441        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6442        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6443        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6444        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6445        let mut configured = http_fixture(&dir, &url, 2);
6446        configured.restart.max_restarts = 2;
6447        let module =
6448            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6449                .supervise_configured_with_health(
6450                    configured.module_spec(),
6451                    true,
6452                    configured.health,
6453                    configured.drain_timeout_ms,
6454                    configured.restart,
6455                )
6456                .unwrap();
6457        let observed = module.clone();
6458        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6459        let served = probes.clone();
6460        let server = tokio::spawn(async move {
6461            loop {
6462                let (mut stream, _) = listener.accept().await.unwrap();
6463                let mut request = [0; 2048];
6464                assert!(stream.read(&mut request).await.unwrap() > 0);
6465                *served
6466                    .lock()
6467                    .unwrap()
6468                    .entry(observed.status().unwrap().spawn_generation)
6469                    .or_default() += 1;
6470                let _ = stream
6471                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6472                    .await;
6473            }
6474        });
6475        timeout(Duration::from_secs(5), async {
6476            while module.state().unwrap() != ModuleState::Failed {
6477                sleep(Duration::from_millis(5)).await;
6478            }
6479        })
6480        .await
6481        .expect("sustained failures must still exhaust the health restart budget");
6482        let status = module.status().unwrap();
6483        server.abort();
6484        assert_eq!(status.restart_count, 2);
6485        assert_eq!(status.lifetime_restarts, 2);
6486        assert!(!status.process_alive);
6487        assert_eq!(
6488            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6489            vec![2, 2, 2]
6490        );
6491    }
6492
6493    #[cfg(unix)]
6494    #[tokio::test]
6495    async fn spawn_clears_the_previous_process_health() {
6496        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6497        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6498        configured.health.cadence = Duration::from_secs(3600);
6499        let module =
6500            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6501                .supervise_configured_with_health(
6502                    configured.module_spec(),
6503                    false,
6504                    configured.health,
6505                    configured.drain_timeout_ms,
6506                    configured.restart,
6507                )
6508                .unwrap();
6509        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6510            state.health.status = SupervisorHealthStatus::Unresponsive;
6511            state.health.consecutive_failures = 3;
6512            state.health.detail = Some("old process refused connections".into());
6513            state.health.metrics = Some(serde_json::json!({"old": true}));
6514            state.health.last_probe_ms = Some(123);
6515            state.health.last_action = Some("restart".into());
6516            state.health.last_action_ms = Some(124);
6517        })
6518        .unwrap();
6519        module.set_enabled(true).await.unwrap();
6520        let health = module.status().unwrap().health;
6521        module.drain().await.unwrap();
6522        assert_eq!(health.consecutive_failures, 0);
6523        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6524        assert_eq!(health.detail, None);
6525        assert_eq!(health.metrics, None);
6526        assert_eq!(health.last_probe_ms, None);
6527        assert_eq!(health.last_action.as_deref(), Some("restart"));
6528        assert_eq!(health.last_action_ms, Some(124));
6529    }
6530
6531    #[cfg(unix)]
6532    #[tokio::test]
6533    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6534        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6535        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6536        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6537        drop(unused);
6538        let configured = http_fixture(&dir, &url, 2);
6539        let module =
6540            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6541                .supervise_configured_with_health(
6542                    configured.module_spec(),
6543                    true,
6544                    configured.health,
6545                    configured.drain_timeout_ms,
6546                    configured.restart,
6547                )
6548                .unwrap();
6549        let before = module.status().unwrap().spawn_generation;
6550        timeout(Duration::from_secs(5), async {
6551            loop {
6552                let status = module.status().unwrap();
6553                if status.spawn_generation > before {
6554                    assert!(status.lifetime_restarts > 0);
6555                    break;
6556                }
6557                sleep(Duration::from_millis(5)).await;
6558            }
6559        })
6560        .await
6561        .expect("sustained HTTP refusal must trigger the health restart policy");
6562        module.drain().await.unwrap();
6563    }
6564
6565    #[cfg(unix)]
6566    #[tokio::test]
6567    async fn http_health_timeout_honours_deadline() {
6568        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6569        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6570        let server = tokio::spawn(async move {
6571            let _held = listener.accept().await.unwrap();
6572            std::future::pending::<()>().await;
6573        });
6574        let error = timeout(
6575            Duration::from_secs(1),
6576            probe_http_health(&url, Duration::from_millis(10)),
6577        )
6578        .await
6579        .expect("the probe must enforce its own deadline")
6580        .unwrap_err();
6581        server.abort();
6582        assert!(error.to_string().contains("timed out"));
6583    }
6584
6585    #[cfg(unix)]
6586    #[tokio::test]
6587    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6588        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6589        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6590        let url = format!(
6591            "http://localhost:{}/healthz",
6592            listener.local_addr().unwrap().port()
6593        );
6594        let server = tokio::spawn(async move {
6595            let (mut stream, _) = listener.accept().await.unwrap();
6596            let mut request = [0u8; 2048];
6597            assert!(stream.read(&mut request).await.unwrap() > 0);
6598            stream
6599                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6600                .await
6601                .unwrap();
6602        });
6603        // The deadline only bounds a hang. A probe that never tried the IPv6
6604        // address would be refused on 127.0.0.1 and fail at once, so a longer
6605        // deadline does not weaken the assertion; one second timed out under a
6606        // loaded parallel test run.
6607        assert_eq!(
6608            probe_http_health(&url, Duration::from_secs(10))
6609                .await
6610                .unwrap()
6611                .status,
6612            HealthStatus::Ok
6613        );
6614        server.await.unwrap();
6615    }
6616
6617    #[cfg(unix)]
6618    #[tokio::test]
6619    async fn http_health_timeout_keeps_partial_status_and_body() {
6620        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6621        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6622        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6623        let server = tokio::spawn(async move {
6624            let (mut stream, _) = listener.accept().await.unwrap();
6625            let mut request = [0u8; 2048];
6626            assert!(stream.read(&mut request).await.unwrap() > 0);
6627            stream
6628                .write_all(
6629                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6630                )
6631                .await
6632                .unwrap();
6633            std::future::pending::<()>().await;
6634        });
6635        let error = probe_http_health(&url, Duration::from_secs(1))
6636            .await
6637            .unwrap_err()
6638            .to_string();
6639        server.abort();
6640        assert!(
6641            error.contains("timed out")
6642                && error.contains("503 Unavailable")
6643                && error.contains("partial diagnostic"),
6644            "{error}"
6645        );
6646    }
6647
6648    #[cfg(unix)]
6649    #[tokio::test]
6650    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6651        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6652        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6653        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6654        let server = tokio::spawn(async move {
6655            let (mut stream, _) = listener.accept().await.unwrap();
6656            let mut request = [0u8; 2048];
6657            assert!(stream.read(&mut request).await.unwrap() > 0);
6658            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6659            let response = format!(
6660                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6661                body.len()
6662            );
6663            stream.write_all(response.as_bytes()).await.unwrap();
6664        });
6665        let error = probe_http_health(&url, Duration::from_secs(1))
6666            .await
6667            .unwrap_err()
6668            .to_string();
6669        server.await.unwrap();
6670        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6671        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6672        assert!(!error.contains("not-in-diagnostic"));
6673    }
6674
6675    #[cfg(unix)]
6676    #[tokio::test]
6677    async fn http_health_real_nats_server_monitoring() {
6678        if std::process::Command::new("nats-server")
6679            .arg("--version")
6680            .env("XDG_DATA_HOME", std::env::temp_dir())
6681            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6682            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6683            .output()
6684            .is_err()
6685        {
6686            eprintln!(
6687                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6688            );
6689            return;
6690        }
6691        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6692        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6693        let port = monitor.local_addr().unwrap().port();
6694        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6695        let client_port = client.local_addr().unwrap().port();
6696        let config = dir.join("server.conf");
6697        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6698        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6699        configured.program = PathBuf::from("nats-server");
6700        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6701        drop(monitor);
6702        drop(client);
6703        let module =
6704            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6705                .supervise_configured_with_health(
6706                    configured.module_spec(),
6707                    true,
6708                    configured.health,
6709                    configured.drain_timeout_ms,
6710                    configured.restart,
6711                )
6712                .unwrap();
6713        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6714        module.drain().await.unwrap();
6715    }
6716
6717    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6718    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6719        let supervisor =
6720            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6721        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6722        let initial = ModuleSpec {
6723            module_id: "rescan-preserves-spawn-facts".to_string(),
6724            program: PathBuf::from("/spawned/module"),
6725            args: Vec::new(),
6726            env: Vec::new(),
6727            reserved: false,
6728            reserved_prefixes: Vec::new(),
6729            protocol: ModuleProtocol::Subc,
6730            overlap: Default::default(),
6731        };
6732        let module = supervisor.supervised_module(
6733            initial.clone(),
6734            supervisor.runtime_config(),
6735            snapshot,
6736            None,
6737        );
6738        let before = module.status().unwrap();
6739        let mut replacement = initial;
6740        replacement.program = PathBuf::from("/rescanned/replacement-module");
6741
6742        module
6743            .update_configuration(replacement, HealthConfig::default(), None)
6744            .await
6745            .unwrap();
6746
6747        let after = module.status().unwrap();
6748        assert_eq!(after.pid, before.pid);
6749        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6750        assert_eq!(after.spawned_from, before.spawned_from);
6751        drop(module);
6752    }
6753}
6754
6755fn unix_ms_now() -> u64 {
6756    SystemTime::now()
6757        .duration_since(UNIX_EPOCH)
6758        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6759        .unwrap_or(0)
6760}
6761
6762async fn supervise_loop(
6763    mut spec: ModuleSpec,
6764    mut runtime: SupervisorRuntimeConfig,
6765    registry: Arc<Registry>,
6766    process_liveness: Arc<SupervisorProcessLiveness>,
6767    snapshot: SharedSnapshot,
6768    mut child: Option<SupervisedChild>,
6769    mut commands: mpsc::Receiver<SupervisorCommand>,
6770) {
6771    let mut health_probe = HealthProbeRuntime::default();
6772    // Registry writes (including embedded callers) notify this module only.
6773    // Subscribe before the first refresh; watch retains changes that arrive
6774    // while commands or probes are running. No polling fallback is needed.
6775    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6776        Ok(changes) => Some(changes),
6777        Err(err) => {
6778            // A poisoned registry cannot accept further writes, so it cannot
6779            // recover via a timer. Keep exit and command handling alive.
6780            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6781            None
6782        }
6783    };
6784    // All restart backoffs run here, including health and operator requests.
6785    // While one is pending the loop serves commands, so disable or drain can
6786    // cancel the replacement without spawning a process just to stop it.
6787    let mut pending_respawn: Option<PendingRespawn> = None;
6788    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6789    // before anything else so a stop that interrupted a swap runs at once.
6790    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6791    let mut previous_instance_recheck = tokio::time::interval_at(
6792        Instant::now() + Duration::from_secs(5),
6793        Duration::from_secs(5),
6794    );
6795    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6796    loop {
6797        #[cfg(test)]
6798        {
6799            lock_snapshot(&snapshot).unwrap().actor_select = None;
6800        }
6801        #[cfg(target_os = "macos")]
6802        if let Some(active) = child.as_mut() {
6803            active.confirm_privacy_exec().await;
6804        }
6805        if let Some(scheduled) = runtime
6806            .scheduled_respawn
6807            .lock()
6808            .unwrap_or_else(|p| p.into_inner())
6809            .take()
6810        {
6811            runtime.recovery_is_daemon = scheduled.daemon_initiated;
6812            pending_respawn = Some(scheduled);
6813        }
6814        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6815            pending_respawn = None;
6816            cancel_deferred_reload(
6817                &runtime,
6818                &spec.module_id,
6819                "respawn cancelled by a supervisor command",
6820            );
6821        }
6822        if child.is_none()
6823            && pending_respawn.is_none()
6824            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6825        {
6826            cancel_deferred_reload(
6827                &runtime,
6828                &spec.module_id,
6829                "respawn cancelled before a replacement was spawned",
6830            );
6831            let mut stranded = false;
6832            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6833                state.respawn_pending = false;
6834                state.coalesced_restart_pending = false;
6835                if matches!(
6836                    state.state,
6837                    ModuleState::Restarting
6838                        | ModuleState::Starting
6839                        | ModuleState::Draining
6840                        | ModuleState::Unresponsive
6841                ) {
6842                    error!(module_id = %spec.module_id, state = ?state.state,
6843                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6844                    stranded = true;
6845                }
6846            });
6847            if stranded {
6848                fail_snapshot(
6849                    &snapshot,
6850                    Some(&spec.module_id),
6851                    None,
6852                    &runtime.failure_context(runtime.recovery_is_daemon),
6853                );
6854            }
6855        }
6856        if let Some(command) = requeued.pop_front() {
6857            if !handle_supervisor_command(
6858                command,
6859                &mut spec,
6860                &mut runtime,
6861                &registry,
6862                &process_liveness,
6863                &snapshot,
6864                &mut child,
6865                &mut commands,
6866                &mut requeued,
6867            )
6868            .await
6869            {
6870                return;
6871            }
6872            if child.is_some() || !respawn_still_pending(&snapshot) {
6873                pending_respawn = None;
6874                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6875                    state.respawn_pending = false
6876                });
6877            }
6878            continue;
6879        }
6880        if child.is_some() {
6881            runtime.recovery_is_daemon = true;
6882            #[cfg(test)]
6883            {
6884                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6885            }
6886            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6887            let active = child.as_mut().expect("child checked above");
6888            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6889                active.registration_deadline = None;
6890            }
6891            let registration_deadline = active.registration_deadline;
6892            let no_checks = runtime.health.http.is_none()
6893                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6894                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6895            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6896                state
6897                    .failed_retry
6898                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
6899            });
6900            let wake_at = match (health_probe.wake_at(), uptime_reset) {
6901                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6902                (probe, None) => probe,
6903            };
6904            let wake_at = match (wake_at, registration_deadline) {
6905                (Some(wake), Some(register)) => Some(wake.min(register)),
6906                (wake, register) => wake.or(register),
6907            };
6908            #[cfg(test)]
6909            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6910            let probe_sleep = supervisor_sleep(wake_at);
6911            tokio::pin!(probe_sleep);
6912            // A `protocol: "none"` child (a plain process that never registers
6913            // over the subc wire, such as nats-server) with no HTTP health
6914            // check has nothing to probe, so it waits only for its exit or a
6915            // supervisor command. A rescan that adds an HTTP check is a command.
6916            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6917            #[cfg(test)]
6918            {
6919                let registration_pending = wire_child
6920                    && registration_changes
6921                        .as_ref()
6922                        .is_some_and(|changes| changes.has_changed().unwrap());
6923                let mut state = lock_snapshot(&snapshot).unwrap();
6924                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6925                    generation: state.spawn_generation,
6926                    turn: state.actor_turns,
6927                    registered_connection: health_probe.registered_connection,
6928                    next_probe_at: health_probe.next_probe_at,
6929                    wake_after,
6930                });
6931            }
6932            let active_child = child.as_mut().expect("child checked above");
6933            tokio::select! {
6934                wait_result = active_child.wait() => {
6935                    if let Ok(mut state) = lock_snapshot(&snapshot) {
6936                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6937                    }
6938                    #[cfg(test)]
6939                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6940                    // Every arm below that gives up on the CHILD must keep the
6941                    // supervision task itself alive (child = None, loop
6942                    // continues into command-serving mode). Returning here
6943                    // closes the command channel, which makes the module
6944                    // permanently unrestartable in-band: a clean child exit
6945                    // of an enabled module once wedged the fleet this way
6946                    // ('supervisor command channel is closed') and required a
6947                    // full daemon restart to recover.
6948                    let exit_report = match wait_result {
6949                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6950                        Err(err) => {
6951                            active_child.drain_stderr(&spec.module_id).await;
6952                            fail_after_wait_error(
6953                                &spec.module_id,
6954                                &snapshot,
6955                                &runtime.terminal_ring,
6956                                &runtime.spawn_events,
6957                                format!("failed to wait for supervised module: {err}"),
6958                            &runtime.failure_context(true),
6959                            );
6960                            untrack_if_registration_released(
6961                                &process_liveness,
6962                                &registry,
6963                                &spec.module_id,
6964                                &snapshot,
6965                            );
6966                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6967                            child = None;
6968                            continue;
6969                        }
6970                    };
6971                    active_child.drain_stderr(&spec.module_id).await;
6972
6973                    let next = on_child_exit(
6974                        &spec,
6975                        runtime.restart_policy,
6976                        &registry,
6977                        &snapshot,
6978                        &runtime.terminal_ring,
6979                        &runtime.spawn_events,
6980                        &runtime.child_roster,
6981                        exit_report,
6982                        true,
6983                    ).await;
6984                    runtime.start_requester = None;
6985                    runtime.pending_activity_restart = false;
6986                    // The exit is recorded, so a daemon shutdown may stop
6987                    // waiting for this child (see `SupervisedChild::wait`).
6988                    active_child.release_roster();
6989                    match next {
6990                        NextAction::Stop { registration_released } => {
6991                            if registration_released {
6992                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6993                            }
6994                            child = None;
6995                        }
6996                        NextAction::Restart { schedule } => {
6997                            let delay = schedule.map_or(
6998                                runtime.restart_policy.delay_for_restart(0),
6999                                |schedule| schedule.delay,
7000                            );
7001                            if let Some(schedule) = schedule {
7002                                log_crash_respawn(&spec.module_id, schedule);
7003                            }
7004                            // The exited child is fully recorded at this point,
7005                            // so release it and count the backoff down in the
7006                            // command-serving branch below rather than sleeping
7007                            // here: commands cannot be received from inside this
7008                            // select arm, and an operator disable or drain that
7009                            // arrives during the backoff must cancel the pending
7010                            // respawn instead of waiting for it to spawn first.
7011                            child = None;
7012                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7013                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7014                        }
7015                    }
7016                }
7017                command = commands.recv() => {
7018                    #[cfg(test)]
7019                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7020                    let Some(command) = command else {
7021                        return;
7022                    };
7023                    if !handle_supervisor_command(
7024                        command,
7025                        &mut spec,
7026                        &mut runtime,
7027                        &registry,
7028                        &process_liveness,
7029                        &snapshot,
7030                        &mut child,
7031                        &mut commands,
7032                        &mut requeued,
7033                    ).await {
7034                        return;
7035                    }
7036                }
7037                _ = async {
7038                    match registration_changes.as_mut() {
7039                        Some(changes) => { let _ = changes.changed().await; }
7040                        None => std::future::pending().await,
7041                    }
7042                }, if wire_child => {
7043                    #[cfg(test)]
7044                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7045                }
7046                _ = &mut probe_sleep => {
7047                    #[cfg(test)]
7048                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7049                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7050                        // Recheck after the wake: HELLO, exit or shutdown may
7051                        // have become ready in the same executor tick.
7052                        active_child.registration_deadline = None;
7053                        if !runtime.child_roster.is_closed()
7054                            && !active_child.registered_since_spawn()
7055                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7056                        {
7057                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7058                                wait_secs = runtime.register_timeout.as_secs(),
7059                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7060                            active_child.spawn_failure = Some("registration_timeout".into());
7061                            if let Err(error) = stop_unregistered_child(active_child).await {
7062                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7063                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7064                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7065                                    &runtime.failure_context(true));
7066                                child = None;
7067                            }
7068                            // The ordinary exit handler records the child's exit,
7069                            // spends crash budget, and schedules recovery.
7070                            continue;
7071                        }
7072                    }
7073                    if health_probe.due() {
7074                        run_health_probe_cycle(
7075                            &spec,
7076                            &runtime,
7077                            &registry,
7078                            &process_liveness,
7079                            &snapshot,
7080                            &mut child,
7081                        ).await;
7082                        if child.is_none() {
7083                            runtime.start_requester = None;
7084                            runtime.pending_activity_restart = false;
7085                        }
7086                        if child.is_some() {
7087                            health_probe.schedule_next(&spec, runtime.health.cadence);
7088                        }
7089                    }
7090                }
7091            }
7092        } else if let Some(pending) = pending_respawn {
7093            tokio::select! {
7094                _ = sleep_until(pending.deadline) => {
7095                    pending_respawn = None;
7096                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7097                    // A command handled below while the backoff elapsed may
7098                    // have stopped the module; never respawn past an operator's
7099                    // disable or drain.
7100                    if !respawn_still_pending(&snapshot) {
7101                        continue;
7102                    }
7103                    // The daemon began shutting down during the backoff: the
7104                    // spawn would be refused anyway, and refusing it here
7105                    // leaves the module stopped instead of reporting a
7106                    // failed restart.
7107                    if runtime.child_roster.is_closed() {
7108                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7109                            state.state = ModuleState::Stopped;
7110                        });
7111                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7112                        continue;
7113                    }
7114                    if let Err(err) = release_dead_registration(
7115                        &registry,
7116                        runtime.forwarding.as_deref(),
7117                        &snapshot,
7118                        &spec.module_id,
7119                        &runtime.failure_context(runtime.recovery_is_daemon),
7120                    ).await {
7121                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7122                        continue;
7123                    }
7124
7125                    if matches!(pending.kind, RespawnKind::Reload) {
7126                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7127                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7128                        if result.is_ok() && runtime.pending_activity_restart {
7129                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7130                        }
7131                        runtime.pending_activity_restart = false;
7132                        if let Some(reply) = reply { let _ = reply.send(result); }
7133                        continue;
7134                    }
7135                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7136                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7137                        Ok(next_child) => {
7138                            child = Some(next_child);
7139                            if runtime.pending_activity_restart {
7140                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7141                                runtime.pending_activity_restart = false;
7142                            }
7143                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7144                        }
7145                        Err(err) => {
7146                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7147                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7148                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7149                        }
7150                    }
7151                }
7152                command = commands.recv() => {
7153                    let Some(command) = command else {
7154                        return;
7155                    };
7156                    if !handle_supervisor_command(
7157                        command,
7158                        &mut spec,
7159                        &mut runtime,
7160                        &registry,
7161                        &process_liveness,
7162                        &snapshot,
7163                        &mut child,
7164                        &mut commands,
7165                        &mut requeued,
7166                    ).await {
7167                        return;
7168                    }
7169                    // Reconcile the pending respawn with what the command did:
7170                    // a start may already have spawned a fresh child,
7171                    // while a disable or drain moved the snapshot out of the
7172                    // state the respawn was counting down from.
7173                    if child.is_some() || !respawn_still_pending(&snapshot) {
7174                        pending_respawn = None;
7175                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7176                    }
7177                }
7178            }
7179        } else {
7180            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7181                (state.enabled && state.state == ModuleState::Failed)
7182                    .then_some(state.failed_retry.pending)
7183                    .flatten()
7184            });
7185            let command = if lock_snapshot(&snapshot)
7186                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7187            {
7188                tokio::select! {
7189                    command = commands.recv() => command,
7190                    _ = previous_instance_recheck.tick() => {
7191                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7192                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7193                            if enabled && !runtime.child_roster.is_closed() {
7194                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7195                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7196                                    Ok(next_child) => child = Some(next_child),
7197                                    Err(error) => {
7198                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7199                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7200                                    }
7201                                }
7202                            } else {
7203                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7204                                    state.waiting_for_previous_pid = None;
7205                                    state.waiting_for_previous_detail = None;
7206                                });
7207                            }
7208                        }
7209                        continue;
7210                    }
7211                }
7212            } else {
7213                tokio::select! {
7214                    biased;
7215                    command = commands.recv() => command,
7216                    _ = async {
7217                        match slow_retry {
7218                            Some(retry) => sleep_until(retry.deadline).await,
7219                            None => std::future::pending().await,
7220                        }
7221                    } => {
7222                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7223                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7224                            state.failed_retry.pending = None;
7225                        });
7226                        if runtime.child_roster.is_closed() {
7227                            continue;
7228                        }
7229                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7230                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7231                        if let Err(error) = set_child_enabled(
7232                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7233                            &runtime.failure_context(true),
7234                        ).await {
7235                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7236                        }
7237                        continue;
7238                    }
7239                }
7240            };
7241            let Some(command) = command else {
7242                return;
7243            };
7244            if !handle_supervisor_command(
7245                command,
7246                &mut spec,
7247                &mut runtime,
7248                &registry,
7249                &process_liveness,
7250                &snapshot,
7251                &mut child,
7252                &mut commands,
7253                &mut requeued,
7254            )
7255            .await
7256            {
7257                return;
7258            }
7259        }
7260    }
7261}
7262
7263// Preserve the deadline until the future is polled. Starting a relative sleep
7264// after the executor yields would restart part of the countdown.
7265async fn supervisor_sleep(wake_at: Option<Instant>) {
7266    match wake_at {
7267        Some(deadline) => sleep_until(deadline).await,
7268        None => std::future::pending().await,
7269    }
7270}
7271
7272fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7273    info!(
7274        module_id,
7275        restart_in_window = schedule.restart_in_window,
7276        delay_ms = schedule.delay.as_millis() as u64,
7277        "respawning after crash"
7278    );
7279}
7280
7281/// Whether the respawn a backoff was counting down to is still wanted. A
7282/// disable or drain handled while the backoff elapsed moves the snapshot out
7283/// of `Restarting`, and the operator's stop must win over the pending respawn,
7284/// so every sleep-then-spawn path re-validates against the live snapshot
7285/// instead of assuming the state it left behind still holds.
7286fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7287    matches!(
7288        lock_snapshot(snapshot),
7289        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7290    )
7291}
7292
7293enum NextAction {
7294    Stop {
7295        registration_released: bool,
7296    },
7297    Restart {
7298        schedule: Option<CrashRestartSchedule>,
7299    },
7300}
7301
7302#[allow(clippy::too_many_arguments)]
7303async fn handle_supervisor_command(
7304    command: SupervisorCommand,
7305    spec: &mut ModuleSpec,
7306    runtime: &mut SupervisorRuntimeConfig,
7307    registry: &Arc<Registry>,
7308    process_liveness: &SupervisorProcessLiveness,
7309    snapshot: &SharedSnapshot,
7310    child: &mut Option<SupervisedChild>,
7311    commands: &mut mpsc::Receiver<SupervisorCommand>,
7312    requeued: &mut VecDeque<SupervisorCommand>,
7313) -> bool {
7314    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7315        runtime.recovery_is_daemon = false;
7316    }
7317    if matches!(
7318        command,
7319        SupervisorCommand::Drain { .. }
7320            | SupervisorCommand::Retire { .. }
7321            | SupervisorCommand::SetEnabled { .. }
7322            | SupervisorCommand::Restart { .. }
7323    ) {
7324        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7325            state.failed_retry.cancel()
7326        });
7327    }
7328    let command =
7329        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7330            match command {
7331                SupervisorCommand::Restart { reply, .. }
7332                | SupervisorCommand::Reload { reply, .. }
7333                | SupervisorCommand::Swap { reply, .. } => {
7334                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7335                    return true;
7336                }
7337                SupervisorCommand::SetEnabled {
7338                    enabled: true,
7339                    reply,
7340                    ..
7341                } => {
7342                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7343                    return true;
7344                }
7345                other => other,
7346            }
7347        } else {
7348            command
7349        };
7350    match command {
7351        SupervisorCommand::Drain { requester, reply } => {
7352            // A plain stop runs no forwarding drain, so nothing reaches the
7353            // module over its connection before the wait: ask by signal.
7354            let result = drain_optional_child(
7355                &spec.module_id,
7356                spec.protocol,
7357                StopNotice::NotSent,
7358                registry,
7359                runtime.forwarding.as_deref(),
7360                snapshot,
7361                &runtime.terminal_ring,
7362                &runtime.spawn_events,
7363                child,
7364                runtime.drain_timeout,
7365                ModuleState::Stopped,
7366                None,
7367                &FailureContext::operator(),
7368            )
7369            .await;
7370            let registration_released = result.is_ok();
7371            if registration_released {
7372                runtime.record_activity_transition(
7373                    "subc.module_stopped",
7374                    &spec.module_id,
7375                    requester,
7376                );
7377            }
7378            let _ = reply.send(result);
7379            if registration_released {
7380                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7381            }
7382            false
7383        }
7384        SupervisorCommand::Retire { requester, reply } => {
7385            let result = async {
7386                let stop_notice = begin_forwarding_drain_if_configured(
7387                    spec,
7388                    runtime,
7389                    registry,
7390                    snapshot,
7391                    None,
7392                    RouteCloseReason::Disable,
7393                )
7394                .await?;
7395                drain_optional_child(
7396                    &spec.module_id,
7397                    spec.protocol,
7398                    stop_notice,
7399                    registry,
7400                    runtime.forwarding.as_deref(),
7401                    snapshot,
7402                    &runtime.terminal_ring,
7403                    &runtime.spawn_events,
7404                    child,
7405                    runtime.drain_timeout,
7406                    ModuleState::Stopped,
7407                    None,
7408                    &FailureContext::operator(),
7409                )
7410                .await
7411            }
7412            .await;
7413            let registration_released = result.is_ok();
7414            if registration_released {
7415                runtime.record_activity_transition(
7416                    "subc.module_stopped",
7417                    &spec.module_id,
7418                    requester,
7419                );
7420            }
7421            let _ = reply.send(result);
7422            if registration_released {
7423                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7424            }
7425            false
7426        }
7427        SupervisorCommand::Restart {
7428            requester,
7429            drain_timeout_ms,
7430            received_at_generation,
7431            queued_at,
7432            reply,
7433        } => {
7434            // Without this line a restart that waited in the queue (behind a
7435            // health probe cycle or another command) was invisible: the log
7436            // showed only the drain timing out, minutes after the operator's call.
7437            info!(
7438                module_id = %spec.module_id,
7439                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7440                "restart command dequeued"
7441            );
7442            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7443            // caller whose own request lane rides the module being restarted: the
7444            // caller's in-flight request keeps the drain from quiescing, the drain
7445            // keeps the restart from completing, and the completion keeps the reply
7446            // from releasing the caller — so the drain always timed out and cut the
7447            // initiator with a GOODBYE, even on a healthy module. Replying once the
7448            // restart is validated lets a self-lane caller settle, which is exactly
7449            // what makes the drain succeed. Completion is observable via
7450            // supervisor.list / module status; a post-ack failure lands the module
7451            // in a visible terminal state below rather than in a reply nobody can
7452            // receive.
7453            let validation = match lock_snapshot(snapshot) {
7454                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7455                    module_id: spec.module_id.clone(),
7456                }),
7457                Ok(_) => Ok(()),
7458                Err(err) => Err(err),
7459            };
7460            let initiated = validation.is_ok();
7461            let _ = reply.send(validation);
7462            // A restart asks for a fresh process. Commands run one at a time,
7463            // so a restart queued behind another restart (two operator calls
7464            // in quick succession) is dequeued the moment the first one has
7465            // spawned its replacement -- before that process has sent HELLO.
7466            // Running it would drain and kill the process the first restart
7467            // just produced, which is the opposite of what both callers asked
7468            // for. If a process spawned after this request was received is
7469            // still supervised, the request is already satisfied. Not when the
7470            // configuration changed since that spawn: then the newer process
7471            // predates the spec this restart may exist to apply.
7472            // Generation publication can lag a child that is already serving
7473            // while its identity is recorded. Only process creation after request
7474            // receipt satisfies a restart, not publication after receipt.
7475            let satisfied_by_generation = if initiated
7476                && child
7477                    .as_ref()
7478                    .is_some_and(|child| child.spawned_at > queued_at)
7479            {
7480                lock_snapshot(snapshot).ok().and_then(|state| {
7481                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7482                })
7483            } else {
7484                None
7485            };
7486            let satisfied_by_pending = initiated
7487                && child.is_none()
7488                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7489                    // No child exists yet, so this replacement will necessarily
7490                    // be created after the queued request, unlike a published
7491                    // generation whose process may already have been serving.
7492                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7493                    if pending {
7494                        state.coalesced_restart_pending = true;
7495                    }
7496                    pending
7497                });
7498            if satisfied_by_pending {
7499                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7500            } else if let Some(generation) = satisfied_by_generation {
7501                info!(
7502                    module_id = %spec.module_id,
7503                    received_at_generation,
7504                    "restart already satisfied by generation {generation}; not restarting again"
7505                );
7506            } else if initiated {
7507                // Precedence: this restart's operator override, else the module's
7508                // configured budget (already resolved into the runtime).
7509                let drain_timeout = drain_timeout_ms
7510                    .map(Duration::from_millis)
7511                    .unwrap_or(runtime.drain_timeout);
7512                if let Err(err) = restart_child(
7513                    spec,
7514                    runtime,
7515                    registry,
7516                    process_liveness,
7517                    snapshot,
7518                    child,
7519                    drain_timeout,
7520                )
7521                .await
7522                {
7523                    warn!(
7524                        module_id = %spec.module_id,
7525                        error = %err,
7526                        "operator restart failed after initiation ack; module state carries the outcome"
7527                    );
7528                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7529                        state.state = ModuleState::Failed;
7530                        clear_current_process_facts(state);
7531                    });
7532                } else {
7533                    runtime.start_requester = requester;
7534                    runtime.pending_activity_restart = true;
7535                }
7536            }
7537            true
7538        }
7539        SupervisorCommand::Reload { requester, reply } => {
7540            let result =
7541                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7542            if result.is_ok() {
7543                runtime.start_requester = requester;
7544                runtime.pending_activity_restart = true;
7545            }
7546            if result.is_ok()
7547                && runtime
7548                    .scheduled_respawn
7549                    .lock()
7550                    .unwrap_or_else(|p| p.into_inner())
7551                    .as_ref()
7552                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7553            {
7554                *runtime
7555                    .deferred_reload_reply
7556                    .lock()
7557                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7558            } else {
7559                let _ = reply.send(result);
7560            }
7561            true
7562        }
7563        SupervisorCommand::SetEnabled {
7564            enabled,
7565            requester,
7566            reply,
7567        } => {
7568            let previous_requester = runtime.start_requester.clone();
7569            if enabled {
7570                runtime.start_requester = requester.clone();
7571            }
7572            let result = set_child_enabled(
7573                spec,
7574                runtime,
7575                registry,
7576                process_liveness,
7577                snapshot,
7578                child,
7579                enabled,
7580                &runtime.failure_context(false),
7581            )
7582            .await;
7583            if matches!(result, Ok(true)) {
7584                runtime.pending_activity_restart = false;
7585                if enabled {
7586                    runtime.start_requester = requester;
7587                } else {
7588                    runtime.start_requester = None;
7589                    runtime.record_activity_transition(
7590                        "subc.module_disabled",
7591                        &spec.module_id,
7592                        requester,
7593                    );
7594                }
7595            } else {
7596                runtime.start_requester = previous_requester;
7597            }
7598            let _ = reply.send(result);
7599            true
7600        }
7601        SupervisorCommand::UpdateConfiguration {
7602            spec: next_spec,
7603            health,
7604            drain_timeout_ms,
7605            reply,
7606        } => {
7607            if let Some(handle) = &runtime.supervisor_handle {
7608                handle.apply_identity_configuration(&next_spec);
7609            }
7610            *spec = next_spec;
7611            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7612                state.configuration_updated_since_spawn = true;
7613            });
7614            let health_changed = runtime.health != health;
7615            runtime.health = health;
7616            // Reset the cadence and old endpoint's failure streak on a live
7617            // health-policy change rather than waiting for its old deadline.
7618            if health_changed {
7619                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7620                    state.health = ModuleHealthStatus::default();
7621                });
7622            }
7623            runtime.drain_timeout = drain_timeout_ms
7624                .map(Duration::from_millis)
7625                .unwrap_or(runtime.default_drain_timeout);
7626            *runtime
7627                .effective_drain_timeout
7628                .lock()
7629                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7630            let _ = reply.send(());
7631            true
7632        }
7633        SupervisorCommand::Swap {
7634            requester,
7635            ready_timeout,
7636            reply,
7637        } => {
7638            let mut swap_runtime = runtime.clone();
7639            swap_runtime.start_requester = requester;
7640            let before = lock_snapshot(snapshot)
7641                .ok()
7642                .map(|state| state.spawn_generation);
7643            let end = swap::run_swap(
7644                spec,
7645                &swap_runtime,
7646                registry,
7647                process_liveness,
7648                snapshot,
7649                child,
7650                commands,
7651                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7652                reply,
7653            )
7654            .await;
7655            if before
7656                != lock_snapshot(snapshot)
7657                    .ok()
7658                    .map(|state| state.spawn_generation)
7659            {
7660                runtime.start_requester = swap_runtime.start_requester;
7661            }
7662            requeued.extend(end.requeue);
7663            true
7664        }
7665    }
7666}
7667
7668async fn restart_child(
7669    spec: &ModuleSpec,
7670    runtime: &SupervisorRuntimeConfig,
7671    registry: &Registry,
7672    process_liveness: &SupervisorProcessLiveness,
7673    snapshot: &SharedSnapshot,
7674    child: &mut Option<SupervisedChild>,
7675    drain_timeout: Duration,
7676) -> Result<(), SuperviseError> {
7677    // Restart cycles a running module; it must not silently start a disabled one.
7678    if !lock_snapshot(snapshot)?.enabled {
7679        return Err(SuperviseError::Disabled {
7680            module_id: spec.module_id.clone(),
7681        });
7682    }
7683    let stop_notice = begin_forwarding_drain_with_timeout(
7684        spec,
7685        runtime,
7686        registry,
7687        snapshot,
7688        None,
7689        RouteCloseReason::Restart,
7690        drain_timeout,
7691    )
7692    .await?;
7693
7694    if child.is_some() {
7695        drain_optional_child(
7696            &spec.module_id,
7697            spec.protocol,
7698            stop_notice,
7699            registry,
7700            runtime.forwarding.as_deref(),
7701            snapshot,
7702            &runtime.terminal_ring,
7703            &runtime.spawn_events,
7704            child,
7705            drain_timeout,
7706            ModuleState::Restarting,
7707            Some(true),
7708            &FailureContext::operator(),
7709        )
7710        .await?;
7711    } else {
7712        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7713            state.enabled = true;
7714            state.state = ModuleState::Restarting;
7715            clear_current_process_facts(state);
7716        })?;
7717        release_dead_registration(
7718            registry,
7719            runtime.forwarding.as_deref(),
7720            snapshot,
7721            &spec.module_id,
7722            &runtime.failure_context(false),
7723        )
7724        .await?;
7725    }
7726
7727    reset_restart_count(snapshot, &spec.module_id)?;
7728    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7729    schedule_respawn(
7730        runtime,
7731        snapshot,
7732        &spec.module_id,
7733        runtime.restart_policy.backoff,
7734        RespawnKind::Spawn,
7735        false,
7736    )
7737}
7738
7739async fn reload_child(
7740    spec: &ModuleSpec,
7741    runtime: &SupervisorRuntimeConfig,
7742    registry: &Registry,
7743    process_liveness: &SupervisorProcessLiveness,
7744    snapshot: &SharedSnapshot,
7745    child: &mut Option<SupervisedChild>,
7746) -> Result<(), SuperviseError> {
7747    // Reload cycles a running module; it must not silently start a disabled one.
7748    if !lock_snapshot(snapshot)?.enabled {
7749        return Err(SuperviseError::Disabled {
7750            module_id: spec.module_id.clone(),
7751        });
7752    }
7753    let stop_notice = begin_forwarding_drain(
7754        spec,
7755        runtime,
7756        registry,
7757        snapshot,
7758        Some(true),
7759        RouteCloseReason::Reload,
7760    )
7761    .await?;
7762
7763    if child.is_some() {
7764        drain_optional_child(
7765            &spec.module_id,
7766            spec.protocol,
7767            stop_notice,
7768            registry,
7769            runtime.forwarding.as_deref(),
7770            snapshot,
7771            &runtime.terminal_ring,
7772            &runtime.spawn_events,
7773            child,
7774            runtime.drain_timeout,
7775            ModuleState::Restarting,
7776            Some(true),
7777            &FailureContext::operator(),
7778        )
7779        .await?;
7780    } else {
7781        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7782            state.enabled = true;
7783            state.state = ModuleState::Restarting;
7784            clear_current_process_facts(state);
7785        })?;
7786        release_dead_registration(
7787            registry,
7788            runtime.forwarding.as_deref(),
7789            snapshot,
7790            &spec.module_id,
7791            &runtime.failure_context(false),
7792        )
7793        .await?;
7794    }
7795
7796    reset_restart_count(snapshot, &spec.module_id)?;
7797    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7798    schedule_respawn(
7799        runtime,
7800        snapshot,
7801        &spec.module_id,
7802        runtime.restart_policy.backoff,
7803        RespawnKind::Reload,
7804        false,
7805    )
7806}
7807
7808async fn finish_reload_child(
7809    spec: &ModuleSpec,
7810    runtime: &SupervisorRuntimeConfig,
7811    registry: &Registry,
7812    process_liveness: &SupervisorProcessLiveness,
7813    snapshot: &SharedSnapshot,
7814    child: &mut Option<SupervisedChild>,
7815) -> Result<(), SuperviseError> {
7816    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7817    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7818        Ok(next_child) => next_child,
7819        Err(err) => {
7820            return handle_reload_spawn_failure(
7821                spec,
7822                runtime,
7823                process_liveness,
7824                snapshot,
7825                child,
7826                format!("new child failed to spawn: {err}"),
7827            )
7828            .await;
7829        }
7830    };
7831    *child = Some(next_child);
7832
7833    let wait_outcome = {
7834        let active_child = child.as_mut().expect("new reload child was just stored");
7835        wait_for_registration_after_reload(
7836            registry,
7837            &spec.module_id,
7838            snapshot,
7839            active_child,
7840            REGISTRY_RELEASE_TIMEOUT,
7841        )
7842        .await?
7843    };
7844
7845    match wait_outcome {
7846        RegistrationWaitOutcome::Registered => {
7847            child
7848                .as_mut()
7849                .expect("registered reload child")
7850                .registration_deadline = None;
7851            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7852            Ok(())
7853        }
7854        RegistrationWaitOutcome::Exited(exit_report) => {
7855            if let Some(active_child) = child.as_mut() {
7856                active_child.drain_stderr(&spec.module_id).await;
7857            }
7858            // Keep the reaped child's roster guard until its terminal is written.
7859            // Shutdown waits on that guard, not on the child Option used for respawn.
7860            let mut exited_child = child.take().expect("exited reload child is still stored");
7861            #[cfg(test)]
7862            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7863                gate.reached.notify_one();
7864                gate.resume.notified().await;
7865            }
7866            let result = handle_reload_child_registration_failure(
7867                spec,
7868                runtime,
7869                registry,
7870                process_liveness,
7871                snapshot,
7872                child,
7873                ReloadRegistrationFailure {
7874                    exit_report: registration_failure_exit_report(exit_report),
7875                    reason: exited_child
7876                        .spawn_failure
7877                        .clone()
7878                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7879                },
7880            )
7881            .await;
7882            exited_child.release_roster();
7883            result
7884        }
7885        RegistrationWaitOutcome::TimedOut => {
7886            let mut timed_out_child = child
7887                .take()
7888                .expect("timed-out reload child is still running");
7889            timed_out_child
7890                .start_kill()
7891                .map_err(|source| SuperviseError::Kill {
7892                    module_id: spec.module_id.clone(),
7893                    source,
7894                })?;
7895            let status = timed_out_child
7896                .wait()
7897                .await
7898                .map_err(|source| SuperviseError::Wait {
7899                    module_id: spec.module_id.clone(),
7900                    source,
7901                })?;
7902            timed_out_child.drain_stderr(&spec.module_id).await;
7903            handle_reload_child_registration_failure(
7904                spec,
7905                runtime,
7906                registry,
7907                process_liveness,
7908                snapshot,
7909                child,
7910                ReloadRegistrationFailure {
7911                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7912                        snapshot,
7913                        &timed_out_child,
7914                        &status,
7915                    )),
7916                    reason: format!(
7917                        "new child did not register within {:?}",
7918                        REGISTRY_RELEASE_TIMEOUT
7919                    ),
7920                },
7921            )
7922            .await
7923        }
7924    }
7925}
7926
7927#[allow(clippy::too_many_arguments)]
7928async fn set_child_enabled(
7929    spec: &ModuleSpec,
7930    runtime: &SupervisorRuntimeConfig,
7931    registry: &Registry,
7932    process_liveness: &SupervisorProcessLiveness,
7933    snapshot: &SharedSnapshot,
7934    child: &mut Option<SupervisedChild>,
7935    enabled: bool,
7936    failure: &FailureContext,
7937) -> Result<bool, SuperviseError> {
7938    let (current_enabled, current_state, respawn_pending) = {
7939        let state = lock_snapshot(snapshot)?;
7940        (state.enabled, state.state, state.respawn_pending)
7941    };
7942    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7943    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7944    // clean (Stopped) has no live process and no other in-band recovery — the
7945    // operator's start is the explicit recovery act and resets the budget. Without
7946    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7947    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7948    // the one providing every agent's shell.
7949    let revive_terminal = enabled
7950        && current_enabled
7951        && child.is_none()
7952        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7953            || (current_state == ModuleState::Restarting && !respawn_pending));
7954    if current_enabled == enabled && !revive_terminal {
7955        return Ok(false);
7956    }
7957
7958    if enabled {
7959        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7960            state.enabled = true;
7961            state.state = ModuleState::Starting;
7962            clear_current_process_facts(state);
7963        })?;
7964        #[cfg(test)]
7965        if runtime.test_seed_stale_facts_before_enable_spawn {
7966            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7967                state.process_alive = true;
7968                state.pid = Some(41);
7969                state.spawned_at_ms = Some(42);
7970                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7971                state.spawned_file_identity = Some(SpawnedFileIdentity {
7972                    device: 43,
7973                    inode: 44,
7974                    #[cfg(windows)]
7975                    windows_image: None,
7976                });
7977            })?;
7978        }
7979        release_dead_registration(
7980            registry,
7981            runtime.forwarding.as_deref(),
7982            snapshot,
7983            &spec.module_id,
7984            failure,
7985        )
7986        .await?;
7987        reset_restart_count(snapshot, &spec.module_id)?;
7988        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7989        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7990            Ok(next_child) => next_child,
7991            Err(err) => {
7992                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7993                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7994                return Err(err);
7995            }
7996        };
7997        *child = Some(next_child);
7998        debug!(module_id = %spec.module_id, "supervised module enabled");
7999        Ok(true)
8000    } else {
8001        let stop_notice = begin_forwarding_drain_if_configured(
8002            spec,
8003            runtime,
8004            registry,
8005            snapshot,
8006            Some(false),
8007            RouteCloseReason::Disable,
8008        )
8009        .await?;
8010        drain_optional_child(
8011            &spec.module_id,
8012            spec.protocol,
8013            stop_notice,
8014            registry,
8015            runtime.forwarding.as_deref(),
8016            snapshot,
8017            &runtime.terminal_ring,
8018            &runtime.spawn_events,
8019            child,
8020            runtime.drain_timeout,
8021            ModuleState::Disabled,
8022            Some(false),
8023            failure,
8024        )
8025        .await?;
8026        debug!(module_id = %spec.module_id, "supervised module disabled");
8027        Ok(true)
8028    }
8029}
8030
8031#[allow(clippy::too_many_arguments)]
8032async fn on_child_exit(
8033    spec: &ModuleSpec,
8034    policy: RestartPolicy,
8035    registry: &Registry,
8036    snapshot: &SharedSnapshot,
8037    terminal_ring: &Arc<Mutex<TerminalRing>>,
8038    spawn_events: &SpawnEventFeed,
8039    roster: &ChildRoster,
8040    exit_report: ExitReport,
8041    daemon_initiated: bool,
8042) -> NextAction {
8043    // Once the daemon has begun shutting down, no exit is a crash to recover
8044    // from: the module is exiting because the daemon is going away (EOF on its
8045    // connection, or a service manager signalling the whole cgroup). Record it
8046    // as such and never schedule a respawn, which would only start a process
8047    // for the shutdown to end again.
8048    if roster.is_closed() {
8049        return on_child_exit_during_daemon_shutdown(
8050            spec,
8051            registry,
8052            snapshot,
8053            terminal_ring,
8054            spawn_events,
8055            exit_report,
8056        )
8057        .await;
8058    }
8059    // Deliberate stops the supervisor asks for (operator stop, disable,
8060    // restart, reload, swap, a health restart, a drain that runs out of budget)
8061    // takes the child out of the supervise loop and reaps it in
8062    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8063    // that reaches this point is either unrequested or a first-HELLO timeout,
8064    // which is deliberately classified as a crash even after a clean TERM exit.
8065    //
8066    // For a subc-wire module a clean exit is still a stop: those modules are
8067    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8068    // crash, and exiting 0 is a deliberate choice the module made. A
8069    // `protocol: "none"` module is a stock program we cannot change, and many
8070    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8071    // stop would leave the module down for good after any stray signal, so it
8072    // goes through the crash path instead: it spends restart budget, respawns
8073    // with the crash backoff, and ends `failed` when the budget runs out.
8074    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8075        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8076    match exit_report.kind {
8077        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8078            info!(
8079                module_id = %spec.module_id,
8080                exit_code = ?exit_report.code,
8081                exit_signal = ?exit_report.signal,
8082                "supervised module exited cleanly"
8083            );
8084            if let Err(err) = publish_after_terminal_record(
8085                &spec.module_id,
8086                || {
8087                    record_terminal(
8088                        &spec.module_id,
8089                        terminal_ring,
8090                        spawn_events,
8091                        &exit_report,
8092                        TerminalDisposition::Stopped,
8093                    )
8094                },
8095                || {
8096                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8097                        state.state = ModuleState::Stopped;
8098                        clear_current_process_facts(state);
8099                        state.last_exit = Some(exit_report.clone());
8100                    })
8101                },
8102            ) {
8103                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8104            }
8105            let registration_released = match wait_for_registration_release(
8106                registry,
8107                &spec.module_id,
8108                REGISTRY_RELEASE_TIMEOUT,
8109            )
8110            .await
8111            {
8112                Ok(()) => true,
8113                Err(err) => {
8114                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8115                    false
8116                }
8117            };
8118            NextAction::Stop {
8119                registration_released,
8120            }
8121        }
8122        ExitKind::Clean | ExitKind::Crash => {
8123            if unrequested_clean_exit_of_protocol_none {
8124                warn!(
8125                    module_id = %spec.module_id,
8126                    exit_code = ?exit_report.code,
8127                    exit_signal = ?exit_report.signal,
8128                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8129                );
8130            } else {
8131                warn!(
8132                    module_id = %spec.module_id,
8133                    exit_code = ?exit_report.code,
8134                    exit_signal = ?exit_report.signal,
8135                    "supervised module exited abnormally (crash)"
8136                );
8137            }
8138            let mut restart_schedule = None;
8139            let mut disposition = TerminalDisposition::Disabled;
8140            let mut final_state = ModuleState::Disabled;
8141            // Set only when the budget is what stopped the module, so the
8142            // terminal record says which limit was hit rather than leaving
8143            // `failed` to be read as "crashed once, badly".
8144            let mut disposition_detail = lock_snapshot(snapshot)
8145                .ok()
8146                .and_then(|mut state| state.spawn_failure.take());
8147            let now = Instant::now();
8148            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8149                if state.enabled {
8150                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8151                        // Reserving the last slot exhausts the budget for the
8152                        // replacement, not for this exit. Publish its scheduled
8153                        // recovery atomically with the reservation so connection
8154                        // cleanup sees it while terminal recording still holds
8155                        // the visible state at Running.
8156                        state.respawn_pending = true;
8157                        final_state = ModuleState::Restarting;
8158                        restart_schedule = Some(schedule);
8159                        disposition = TerminalDisposition::Restarting;
8160                    } else {
8161                        final_state = ModuleState::Failed;
8162                        disposition = TerminalDisposition::Failed;
8163                        let budget = policy.budget_exhausted_detail();
8164                        disposition_detail =
8165                            Some(disposition_detail.take().map_or_else(
8166                                || budget.clone(),
8167                                |cause| format!("{cause}; {budget}"),
8168                            ));
8169                    }
8170                } else {
8171                    disposition = TerminalDisposition::Disabled;
8172                }
8173            }) {
8174                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8175                return NextAction::Stop {
8176                    registration_released: false,
8177                };
8178            }
8179            if disposition == TerminalDisposition::Failed {
8180                // The window is in the message, not only in the fields: this line
8181                // is read in a scrollback where a bare `max_restarts=3` reads as a
8182                // lifetime cap and sends the operator looking for three crashes
8183                // that never happened together.
8184                error!(
8185                    module_id = %spec.module_id,
8186                    max_restarts = policy.max_restarts,
8187                    window_secs = policy.window.as_secs(),
8188                    "module stopped: {}",
8189                    policy.budget_exhausted_detail()
8190                );
8191            }
8192            if let Err(err) = publish_after_terminal_record(
8193                &spec.module_id,
8194                || {
8195                    record_terminal_with_detail(
8196                        &spec.module_id,
8197                        terminal_ring,
8198                        spawn_events,
8199                        &exit_report,
8200                        disposition,
8201                        disposition_detail,
8202                    )
8203                },
8204                || {
8205                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8206                        state.state = final_state;
8207                        clear_current_process_facts(state);
8208                        if final_state == ModuleState::Failed
8209                            && daemon_initiated
8210                            && !roster.is_closed()
8211                        {
8212                            state.failed_retry.schedule(&spec.module_id);
8213                        }
8214                        state.last_exit = Some(exit_report.clone());
8215                    })
8216                },
8217            ) {
8218                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8219            }
8220
8221            if let Some(schedule) = restart_schedule {
8222                NextAction::Restart {
8223                    schedule: Some(schedule),
8224                }
8225            } else {
8226                let registration_released = match wait_for_registration_release(
8227                    registry,
8228                    &spec.module_id,
8229                    REGISTRY_RELEASE_TIMEOUT,
8230                )
8231                .await
8232                {
8233                    Ok(()) => true,
8234                    Err(err) => {
8235                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8236                        false
8237                    }
8238                };
8239                NextAction::Stop {
8240                    registration_released,
8241                }
8242            }
8243        }
8244        ExitKind::DeliberateSeverance => {
8245            warn!(
8246                module_id = %spec.module_id,
8247                exit_code = ?exit_report.code,
8248                exit_signal = ?exit_report.signal,
8249                "supervised module exited after deliberate connection severance"
8250            );
8251            let should_restart = match lock_snapshot(snapshot) {
8252                Ok(state) => state.enabled,
8253                Err(err) => {
8254                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8255                    return NextAction::Stop {
8256                        registration_released: false,
8257                    };
8258                }
8259            };
8260            let final_state = if should_restart {
8261                ModuleState::Restarting
8262            } else {
8263                ModuleState::Disabled
8264            };
8265            if let Err(err) = publish_after_terminal_record(
8266                &spec.module_id,
8267                || {
8268                    record_terminal(
8269                        &spec.module_id,
8270                        terminal_ring,
8271                        spawn_events,
8272                        &exit_report,
8273                        terminal_disposition(final_state),
8274                    )
8275                },
8276                || {
8277                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8278                        state.state = final_state;
8279                        clear_current_process_facts(state);
8280                        state.last_exit = Some(exit_report.clone());
8281                        state.lifetime_restarts += 1;
8282                    })
8283                },
8284            ) {
8285                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8286                return NextAction::Stop {
8287                    registration_released: false,
8288                };
8289            }
8290
8291            if should_restart {
8292                NextAction::Restart { schedule: None }
8293            } else {
8294                let registration_released = match wait_for_registration_release(
8295                    registry,
8296                    &spec.module_id,
8297                    REGISTRY_RELEASE_TIMEOUT,
8298                )
8299                .await
8300                {
8301                    Ok(()) => true,
8302                    Err(err) => {
8303                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8304                        false
8305                    }
8306                };
8307                NextAction::Stop {
8308                    registration_released,
8309                }
8310            }
8311        }
8312    }
8313}
8314
8315async fn on_child_exit_during_daemon_shutdown(
8316    spec: &ModuleSpec,
8317    registry: &Registry,
8318    snapshot: &SharedSnapshot,
8319    terminal_ring: &Arc<Mutex<TerminalRing>>,
8320    spawn_events: &SpawnEventFeed,
8321    exit_report: ExitReport,
8322) -> NextAction {
8323    info!(
8324        module_id = %spec.module_id,
8325        exit_code = ?exit_report.code,
8326        exit_signal = ?exit_report.signal,
8327        exit_kind = ?exit_report.kind,
8328        "supervised module exited during daemon shutdown; not restarting it"
8329    );
8330    if let Err(err) = publish_after_terminal_record(
8331        &spec.module_id,
8332        || {
8333            record_terminal(
8334                &spec.module_id,
8335                terminal_ring,
8336                spawn_events,
8337                &exit_report,
8338                TerminalDisposition::DaemonShutdown,
8339            )
8340        },
8341        || {
8342            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8343                state.state = ModuleState::Stopped;
8344                clear_current_process_facts(state);
8345                state.last_exit = Some(exit_report.clone());
8346            })
8347        },
8348    ) {
8349        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8350    }
8351    let registration_released =
8352        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8353            .await
8354            .is_ok();
8355    NextAction::Stop {
8356        registration_released,
8357    }
8358}
8359
8360// Keep the status lock out of terminal I/O. Readers may observe the previous
8361// state while recording is in progress, but never a completed exit without its
8362// history. Journal errors are logged by the journal and retained in the ring;
8363// a recording panic still publishes the final state before resuming the unwind.
8364fn publish_after_terminal_record<T>(
8365    module_id: &str,
8366    record: impl FnOnce(),
8367    publish: impl FnOnce() -> T,
8368) -> T {
8369    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8370    let published = publish();
8371    if let Err(panic) = recorded {
8372        error!(
8373            module_id,
8374            "terminal recording panicked; published final state before resuming unwind"
8375        );
8376        std::panic::resume_unwind(panic);
8377    }
8378    published
8379}
8380
8381fn fail_after_wait_error(
8382    module_id: &str,
8383    snapshot: &SharedSnapshot,
8384    terminal_ring: &Arc<Mutex<TerminalRing>>,
8385    spawn_events: &SpawnEventFeed,
8386    detail: String,
8387    failure: &FailureContext,
8388) {
8389    let report = wait_error_exit_report();
8390    publish_after_terminal_record(
8391        module_id,
8392        || {
8393            record_terminal_with_detail(
8394                module_id,
8395                terminal_ring,
8396                spawn_events,
8397                &report,
8398                TerminalDisposition::Failed,
8399                Some(detail),
8400            )
8401        },
8402        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8403    );
8404}
8405
8406#[cfg(test)]
8407fn record_wait_error_terminal(
8408    module_id: &str,
8409    terminal_ring: &Arc<Mutex<TerminalRing>>,
8410    spawn_events: &SpawnEventFeed,
8411) {
8412    record_terminal(
8413        module_id,
8414        terminal_ring,
8415        spawn_events,
8416        &wait_error_exit_report(),
8417        TerminalDisposition::Failed,
8418    );
8419}
8420
8421fn record_terminal(
8422    module_id: &str,
8423    terminal_ring: &Arc<Mutex<TerminalRing>>,
8424    spawn_events: &SpawnEventFeed,
8425    exit_report: &ExitReport,
8426    disposition: TerminalDisposition,
8427) {
8428    record_terminal_with_detail(
8429        module_id,
8430        terminal_ring,
8431        spawn_events,
8432        exit_report,
8433        disposition,
8434        None,
8435    );
8436}
8437
8438/// The ring lock is held only to capture the read (see
8439/// `TerminalJournal::capture_read`), so this module's exits keep recording
8440/// while the journal files are read. Blocking: it reads files.
8441fn durable_terminal_history_of(
8442    terminal_ring: &Mutex<TerminalRing>,
8443    module_id: &str,
8444) -> subc_control::TerminalHistory {
8445    let read = terminal_ring
8446        .lock()
8447        .unwrap_or_else(|p| p.into_inner())
8448        .capture_durable_history();
8449    read.read(module_id)
8450}
8451
8452fn record_terminal_with_detail(
8453    module_id: &str,
8454    terminal_ring: &Arc<Mutex<TerminalRing>>,
8455    spawn_events: &SpawnEventFeed,
8456    exit_report: &ExitReport,
8457    disposition: TerminalDisposition,
8458    disposition_detail: Option<String>,
8459) {
8460    #[cfg(test)]
8461    terminal_history_tests::before_terminal_record(spawn_events);
8462    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8463    let record = TerminalRecord {
8464        exit_code: exit_report.code,
8465        exit_signal: exit_report.signal,
8466        at_ms: exit_report.at_ms,
8467        disposition,
8468        exit_kind: exit_report.kind.into(),
8469        disposition_detail,
8470    };
8471    terminal_ring
8472        .lock()
8473        .unwrap_or_else(|poisoned| poisoned.into_inner())
8474        .record_exit(module_id, record);
8475}
8476
8477fn untrack_if_registration_released(
8478    process_liveness: &SupervisorProcessLiveness,
8479    registry: &Registry,
8480    module_id: &str,
8481    snapshot: &SharedSnapshot,
8482) {
8483    match registry.get_module(module_id) {
8484        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8485        Ok(Some(_)) => {}
8486        Err(err) => {
8487            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8488        }
8489    }
8490}
8491
8492/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8493/// then apply the module's configured entries minus daemon-private capture keys.
8494///
8495/// Separated from `spawn_child` only so it can be asserted without spawning a
8496/// process — a duplicate of this logic in a test would pass while the real one
8497/// drifted, which is the defect class this function exists to avoid.
8498/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8499/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8500/// either and the argument would stop a stock binary from starting at all.
8501/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8502///
8503/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8504/// through [`apply_wire_spawn_args_for_role`].
8505#[cfg(test)]
8506fn apply_wire_spawn_args(
8507    command: &mut Command,
8508    spec: &ModuleSpec,
8509    connection_file_path: Option<&std::path::Path>,
8510    handle: Option<&SupervisorHandle>,
8511) -> Result<Option<NonceHandoff>, SuperviseError> {
8512    apply_wire_spawn_args_for_role(
8513        command,
8514        spec,
8515        connection_file_path,
8516        handle,
8517        SpawnRole::Plain,
8518    )
8519}
8520
8521/// The read end of a spawn's launch-nonce pipe, prepared by
8522/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8523/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8524/// server before spawn and authenticates its client after the child exists.
8525#[cfg(unix)]
8526type NonceHandoff = subc_os::LaunchNonceHandoff;
8527#[cfg(windows)]
8528type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8529#[cfg(not(any(unix, windows)))]
8530type NonceHandoff = std::convert::Infallible;
8531
8532/// Create the launch nonce that a new module process must present when it
8533/// registers or opens a consumer connection, including a replacement process
8534/// started alongside a still-serving module.
8535///
8536/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8537/// a separate candidate token so the still-serving incumbent and its consumers
8538/// keep their nonce. Both records are installed before the process exists, so
8539/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8540///
8541/// On Unix the nonce is delivered only through a pipe. It is written into
8542/// a pipe whose read end the child gets as descriptor 3, named by
8543/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8544/// process of the same user cannot read it with `ps eww`. That handoff is
8545/// returned rather than installed here, because installing it replaces
8546/// whatever the child has at descriptor 3 and so must be the last pre-exec
8547/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8548/// the Linux callback that writes the process into its cgroup. Only after that
8549/// does `handoff.install_last` register the descriptor-3 replacement, so it
8550/// cannot close a descriptor the cgroup callback still needs.
8551/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8552fn apply_wire_spawn_args_for_role(
8553    command: &mut Command,
8554    spec: &ModuleSpec,
8555    connection_file_path: Option<&std::path::Path>,
8556    handle: Option<&SupervisorHandle>,
8557    role: SpawnRole,
8558) -> Result<Option<NonceHandoff>, SuperviseError> {
8559    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8560    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8561    // included: a daemon started from a module's process tree inherits it,
8562    // and passing it on would point the child at a descriptor it does not
8563    // have.
8564    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8565    #[cfg(windows)]
8566    {
8567        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8568        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8569    }
8570    // Remove inherited or configured copies too: withholding must mean absent.
8571    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8572    if spec.protocol == ModuleProtocol::None {
8573        return Ok(None);
8574    }
8575    if let Some(connection_file_path) = connection_file_path {
8576        command.arg(SUBC_ARG).arg(connection_file_path);
8577    }
8578
8579    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8580    // route.open attestation. Reserved modules additionally use the same nonce
8581    // for HELLO id-squatting protection. A respawn rotates both records.
8582    let nonce = generate_launch_nonce()?;
8583    if let Some(handle) = handle {
8584        match role {
8585            SpawnRole::Plain => {
8586                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8587                if spec.reserved {
8588                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8589                }
8590            }
8591            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8592        }
8593    }
8594    #[cfg(unix)]
8595    let handoff = {
8596        let handoff =
8597            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8598                program: spec.program.clone(),
8599                source,
8600                cgroup_path: None,
8601            })?;
8602        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8603        Some(handoff)
8604    };
8605    #[cfg(windows)]
8606    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8607    #[cfg(not(any(unix, windows)))]
8608    let handoff = None;
8609    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8610    // until the daemon's per-spawn delivery records show that every running
8611    // Windows module has read its secret from the named pipe.
8612    #[cfg(not(unix))]
8613    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8614    Ok(handoff)
8615}
8616
8617fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8618    command.env_remove(CK_LOG_ENV);
8619    // The spawn role is the supervisor's to set, and only on a swap candidate
8620    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8621    // it, is what makes it absent on a plain spawn: the daemon's own
8622    // environment could carry it, and so could a spec built outside daemon
8623    // config (config refuses it as an `env` key). A module reading it on a
8624    // plain restart would pick the long swap budget and leave callers waiting.
8625    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8626    for (key, value) in &spec.env {
8627        // cortexkit-log currently exposes retention only as a Rust struct, not
8628        // environment names. These values are daemon-private sink metadata and
8629        // must never become a public child-process contract by being inherited.
8630        if matches!(
8631            key.as_str(),
8632            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8633        ) || key == SUBC_SPAWN_ROLE_ENV
8634        {
8635            continue;
8636        }
8637        command.env(key, value);
8638    }
8639}
8640
8641#[cfg(windows)]
8642fn prepare_windows_nonce_pipe(
8643    command: &mut Command,
8644    spec: &ModuleSpec,
8645    nonce: &str,
8646) -> Option<NonceHandoff> {
8647    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8648        Ok(handoff) => {
8649            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8650            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8651            // still register until every module reads the pipe. A reader without
8652            // this variable set to env returns the pipe error instead of using
8653            // an inherited or configured environment copy.
8654            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8655            Some(handoff)
8656        }
8657        Err(error) => {
8658            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8659            None
8660        }
8661    }
8662}
8663
8664#[cfg(windows)]
8665fn start_windows_nonce_pipe(
8666    handoff: Option<NonceHandoff>,
8667    pid: u32,
8668    spec: &ModuleSpec,
8669) -> Option<subc_os::LaunchNoncePipeDelivery> {
8670    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8671    // same maximum wait used for registration during a reload. Older modules
8672    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8673    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8674        Ok(delivery) => Some(delivery),
8675        Err(error) => {
8676            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8677            None
8678        }
8679    })
8680}
8681
8682fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8683    #[cfg(windows)]
8684    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8685        return Some(
8686            if snapshot
8687                .pipe_consumed
8688                .as_ref()
8689                .is_some_and(|consumed| consumed.get().is_some())
8690            {
8691                "pipe"
8692            } else {
8693                "env"
8694            }
8695            .to_owned(),
8696        );
8697    }
8698    let _ = snapshot;
8699    None
8700}
8701
8702/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8703/// of a blue/green swap.
8704#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8705enum SpawnRole {
8706    Plain,
8707    SwapCandidate,
8708}
8709
8710/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8711/// `apply_child_env` has already removed the variable for every spawn.
8712fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8713    if role == SpawnRole::SwapCandidate {
8714        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8715    }
8716}
8717
8718fn previous_instance_error(
8719    spec: &ModuleSpec,
8720    pid: u32,
8721    reason: crate::live_children::RetainedReason,
8722) -> SuperviseError {
8723    SuperviseError::Spawn {
8724        program: spec.program.clone(),
8725        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8726        cgroup_path: None,
8727    }
8728}
8729
8730fn spawn_child(
8731    spec: &ModuleSpec,
8732    connection_file_path: Option<&std::path::Path>,
8733    handle: Option<&SupervisorHandle>,
8734    ring: &Arc<Mutex<StderrRing>>,
8735    capture_logs_dir: Option<&std::path::Path>,
8736    roster: &ChildRoster,
8737    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8738) -> Result<SupervisedChild, SuperviseError> {
8739    spawn_child_in_slot(
8740        spec,
8741        connection_file_path,
8742        handle,
8743        ring,
8744        capture_logs_dir,
8745        roster,
8746        #[cfg(target_os = "linux")]
8747        cgroup_placement,
8748        SpawnRole::Plain,
8749        false,
8750    )
8751}
8752
8753/// Spawn one process of `spec` into a slot.
8754///
8755/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8756/// A swap candidate needs a different cgroup from the process it is replacing,
8757/// which is still alive: in the same cgroup the two would be one kill domain,
8758/// and killing a failed candidate could take the incumbent with it.
8759///
8760/// The stderr capture file is `<module_id>.stderr.log` for every process of
8761/// the module, whichever slot it is in, because that is the one file
8762/// `ck module logs` reads. During a swap's overlap both processes append to it;
8763/// the daemon writes whole lines, so the two interleave by line, which is also
8764/// the merged view an operator wants while a swap runs.
8765#[allow(clippy::too_many_arguments)]
8766fn spawn_child_in_slot(
8767    spec: &ModuleSpec,
8768    connection_file_path: Option<&std::path::Path>,
8769    handle: Option<&SupervisorHandle>,
8770    ring: &Arc<Mutex<StderrRing>>,
8771    capture_logs_dir: Option<&std::path::Path>,
8772    roster: &ChildRoster,
8773    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8774    role: SpawnRole,
8775    alternate_slot: bool,
8776) -> Result<SupervisedChild, SuperviseError> {
8777    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8778        return Err(previous_instance_error(spec, pid, reason));
8779    }
8780    if roster.is_closed() {
8781        return Err(SuperviseError::Spawn {
8782            program: spec.program.clone(),
8783            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8784            cgroup_path: None,
8785        });
8786    }
8787    #[cfg(target_os = "linux")]
8788    let cgroup_name = {
8789        // Slot names alone are not kill domains: a retired incumbent may still
8790        // be draining when a later enable/restart spawns into the same slot.
8791        // Decimal entropy keeps the suffix unambiguous; Placement performs
8792        // the module-id escaping and constructs the filesystem path.
8793        if cgroup_placement.is_none() {
8794            swap::cgroup_name(&spec.module_id, alternate_slot)
8795        } else {
8796            let nonce = generate_launch_nonce()?;
8797            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8798            // Leave room for byte escaping and the suffix under NAME_MAX. The
8799            // label is only for humans; the nonce identifies the kill domain.
8800            let mut end = spec.module_id.len().min(64);
8801            while !spec.module_id.is_char_boundary(end) {
8802                end -= 1;
8803            }
8804            format!(
8805                "{}_{suffix}",
8806                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8807            )
8808        }
8809    };
8810    #[cfg(not(target_os = "linux"))]
8811    let _ = alternate_slot;
8812    #[cfg(target_os = "macos")]
8813    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8814    #[cfg(windows)]
8815    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8816        program: spec.program.clone(),
8817        source,
8818        cgroup_path: None,
8819    })?;
8820    #[cfg(windows)]
8821    let mut command = Command::new(image_capture.path());
8822    #[cfg(not(any(target_os = "macos", windows)))]
8823    let mut command = Command::new(&spec.program);
8824    command.args(&spec.args);
8825    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8826    // that is the whole of the intent, so remove that one key rather than the
8827    // environment.
8828    //
8829    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8830    // and took the POSIX environment with it. Modules spawned that way had no
8831    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8832    // logging:
8833    //
8834    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8835    //     both unset it fell back to the temp dir alone and `ck` could not find
8836    //     a daemon running on the same machine from inside any module's process
8837    //     tree — reporting a path the file has never lived at, which reads as
8838    //     "the daemon did not write its file".
8839    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8840    //     the RELATIVE `.local/share`, so a module deriving its own store path
8841    //     resolved it against its own CWD. That is the store-fragmentation
8842    //     defect the daemon already refuses in config (`parse_doc` rejects a
8843    //     relative `storage.data_home`) arriving by derivation instead.
8844    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8845    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8846    //     quietly rather than erroring.
8847    //
8848    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8849    // offered one candidate under /tmp while the file sat in /run/user/1000.
8850    //
8851    // A configured module is unaffected either way: `module_spec()` puts the
8852    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8853    // wins over anything ambient.
8854    apply_child_env(&mut command, spec);
8855    apply_spawn_role(&mut command, role);
8856    let nonce_handoff =
8857        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8858
8859    #[cfg(target_os = "linux")]
8860    let cgroup_path = cgroup_placement
8861        .map(|placement| placement.module_path(&cgroup_name))
8862        .transpose()
8863        .map_err(|source| SuperviseError::Cgroup {
8864            module_id: spec.module_id.clone(),
8865            source,
8866        })?;
8867    #[cfg(not(target_os = "linux"))]
8868    let cgroup_path: Option<PathBuf> = None;
8869    #[cfg(target_os = "linux")]
8870    if let Some(path) = &cgroup_path {
8871        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8872            if let Some(placement) = cgroup_placement {
8873                remove_module_cgroup(placement, &cgroup_name);
8874            }
8875            return Err(error);
8876        }
8877    }
8878
8879    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8880        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8881        match ChildOutputSink::open(&path, capture_retention(spec)) {
8882            Ok(sink) => sink,
8883            Err(error) => {
8884                warn!(
8885                    module_id = %spec.module_id,
8886                    path = %path.display(),
8887                    error = %error,
8888                    "could not open child output capture file; forwarding to stderr"
8889                );
8890                ChildOutputSink::Stderr
8891            }
8892        }
8893    } else {
8894        ChildOutputSink::Stderr
8895    };
8896
8897    command.stdout(Stdio::piped());
8898    command.stderr(Stdio::piped());
8899    command.kill_on_drop(true);
8900    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8901    // before exec). In the daemon's group, a service manager that kills the
8902    // job's process group when the daemon exits (launchd's default) killed
8903    // every module at the same moment its control connection closed, so no
8904    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8905    // module is reached only by the daemon: the EOF it sees when its
8906    // connection closes, and the bounded stop in `child_roster` for anything
8907    // still running after that. On Linux this composes with the cgroup
8908    // placement above: that is a pre_exec write to cgroup.procs, std performs
8909    // setpgid in the child before running pre_exec callbacks, and the two
8910    // change independent process attributes.
8911    //
8912    // stdin is /dev/null because a process outside the terminal's foreground
8913    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8914    // by hand would otherwise hand down. Under a service manager stdin is
8915    // already /dev/null.
8916    #[cfg(unix)]
8917    command.process_group(0);
8918    command.stdin(Stdio::null());
8919    // The LAST pre-exec step, after the cgroup placement above: installing the
8920    // nonce at descriptor 3 replaces whatever the child had there, which could
8921    // be the descriptor an earlier step writes through.
8922    #[cfg(unix)]
8923    if let Some(handoff) = nonce_handoff {
8924        handoff.install_last(command.as_std_mut());
8925    }
8926    #[cfg(not(any(unix, windows)))]
8927    let _ = nonce_handoff;
8928
8929    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8930    // cannot run a single instruction -- and therefore cannot spawn a
8931    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8932    // other two steps and why the window matters.
8933    #[cfg(windows)]
8934    {
8935        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8936            program: spec.program.clone(),
8937            source,
8938            cgroup_path: None,
8939        })?;
8940        subc_jobobject::suspend_on_create_async(&mut command);
8941    }
8942    // Timestamp before the spawn call: on Unix the child can serve before that
8943    // call even returns. This conservative bound never labels an older child
8944    // as a replacement for a request received during process creation.
8945    let spawned_at = Instant::now();
8946    let mut child = match command.spawn() {
8947        Ok(child) => child,
8948        Err(source) => {
8949            #[cfg(target_os = "linux")]
8950            if let Some(placement) = cgroup_placement {
8951                remove_module_cgroup(placement, &cgroup_name);
8952            }
8953            return Err(SuperviseError::Spawn {
8954                program: spec.program.clone(),
8955                source,
8956                cgroup_path,
8957            });
8958        }
8959    };
8960    #[cfg(test)]
8961    spawn_record_tests::after_create(&child, spec);
8962    // The parent must close its writer now: the acknowledgement pipe reports EOF
8963    // only when every writer is gone, and the child's copy closes when the
8964    // trampoline replaces itself with the module. Command holds only an integer
8965    // in its pre_exec callback, not another writer.
8966    #[cfg(target_os = "macos")]
8967    drop(exec_ack);
8968
8969    // Bind while the child is still suspended, before job assignment resumes it.
8970    // Keeping the executable handle open across CreateProcessW denies rename
8971    // and deletion, so the file cannot be exchanged during spawn. Bind that
8972    // file ID to the process handle returned by spawn, never a reopened PID.
8973    #[cfg(windows)]
8974    let windows_image = match image_capture.bind_tokio(&child) {
8975        Ok(image) => Arc::new(image),
8976        Err(source) => {
8977            kill_and_reap_rejected_child(&mut child);
8978            return Err(SuperviseError::Spawn {
8979                program: spec.program.clone(),
8980                source,
8981                cgroup_path: None,
8982            });
8983        }
8984    };
8985    // Assign the suspended child to its module job; resume only after its
8986    // process identity has been durably recorded for orphan recovery.
8987    #[cfg(windows)]
8988    let job = contain_spawned_child(&mut child, spec)?;
8989    let spawned_at_ms = unix_ms_now();
8990    #[cfg(not(windows))]
8991    let spawned_from = spec.program.clone();
8992    #[cfg(windows)]
8993    let spawned_from = windows_image.path().to_path_buf();
8994    #[cfg(not(windows))]
8995    let spawned_file_identity = spawned_file_identity(&spawned_from);
8996    #[cfg(windows)]
8997    let spawned_file_identity = Some(SpawnedFileIdentity {
8998        device: 0,
8999        inode: 0,
9000        windows_image: Some(Arc::clone(&windows_image)),
9001    });
9002    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9003        program: spec.program.clone(),
9004        source: io::Error::other("spawned child exposed no live pid"),
9005        cgroup_path: cgroup_path.clone(),
9006    })?;
9007    #[cfg(windows)]
9008    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9009    #[cfg(not(windows))]
9010    let process_start_time = crate::provenance::process_start_time(pid);
9011    #[cfg(windows)]
9012    let process_start_time = Some(windows_image.start_time());
9013    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9014    #[cfg(all(test, target_os = "macos"))]
9015    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9016    // Record the committed kernel image, not the configured pathname: PATH
9017    // lookup and shebang interpretation can select a different executable.
9018    // Linux's posix_spawn can return before that image is installed, so the
9019    // observer also waits for the kernel's first-exec flag to clear.
9020    #[cfg(not(windows))]
9021    let recorded_image = observe_spawned_image(pid);
9022    // spawn() confirms only the first exec, into the trampoline. Never persist
9023    // the trampoline image; the asynchronous acknowledgement publishes the
9024    // module image once the trampoline has replaced itself with the module.
9025    #[cfg(target_os = "macos")]
9026    let recorded_image = if privacy_exec.is_some() {
9027        None
9028    } else {
9029        recorded_image
9030    };
9031    #[cfg(target_os = "linux")]
9032    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9033    #[cfg(not(target_os = "linux"))]
9034    let recorded_cgroup_name = None;
9035    let roster_guard = roster.admit(
9036        spec.module_id.clone(),
9037        pid,
9038        spec.protocol,
9039        process_start_time,
9040        crate::child_roster::RecordedIdentity {
9041            #[cfg(not(windows))]
9042            start_time: recorded_image.map(|image| image.start_time),
9043            #[cfg(windows)]
9044            start_time: process_start_time,
9045            #[cfg(not(windows))]
9046            executable: recorded_image
9047                .and_then(|image| image.executable)
9048                .map(crate::live_children::ExecutableIdentity::from),
9049            #[cfg(windows)]
9050            executable: Some(crate::live_children::ExecutableIdentity::from(
9051                windows_image.identity(),
9052            )),
9053            cgroup_name: recorded_cgroup_name,
9054            #[cfg(target_os = "linux")]
9055            cgroup_placement: cgroup_placement.cloned(),
9056        },
9057    );
9058    let roster_guard = match roster_guard {
9059        Ok(guard) => guard,
9060        Err(failure) => {
9061            return failure.with_cleanup(|source| {
9062                #[cfg(windows)]
9063                if let Some(job) = &job {
9064                    let _ = job.terminate();
9065                }
9066                #[cfg(target_os = "linux")]
9067                kill_module_cgroup(cgroup_placement, &cgroup_name);
9068                kill_and_reap_rejected_child(&mut child);
9069                #[cfg(target_os = "linux")]
9070                if let Some(placement) = cgroup_placement {
9071                    remove_module_cgroup(placement, &cgroup_name);
9072                }
9073                Err(SuperviseError::Spawn {
9074                    program: spec.program.clone(),
9075                    source: io::Error::other(format!(
9076                        "could not durably record module child: {source}"
9077                    )),
9078                    cgroup_path,
9079                })
9080            });
9081        }
9082    };
9083    // The pre-spawn check can pass just before shutdown begins. Ownership is
9084    // published at the start of admission, even while a record write is queued.
9085    // If shutdown already saw an empty roster before that publication, this
9086    // check sees the roster closed and ends the child before starting a module.
9087    if roster.is_closed() {
9088        // This child was never admitted, so there is no module protocol shutdown to wait for.
9089        #[cfg(target_os = "linux")]
9090        kill_module_cgroup(cgroup_placement, &cgroup_name);
9091        if let Err(error) = child.start_kill() {
9092            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9093        }
9094        #[cfg(target_os = "linux")]
9095        if let Some(placement) = cgroup_placement {
9096            // This spawn was never admitted, so shutdown has no roster entry
9097            // to await. Do not detach its cleanup: the runtime could exit
9098            // before that task reaps the rejected child and removes its group.
9099            while matches!(child.try_wait(), Ok(None)) {
9100                std::thread::yield_now();
9101            }
9102            if matches!(
9103                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9104                subc_cgroup::KillOutcome::Killed
9105            ) {
9106                if let Ok(path) = placement.module_path(&cgroup_name) {
9107                    while std::fs::read_to_string(path.join("cgroup.events"))
9108                        .ok()
9109                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9110                    {
9111                        std::thread::yield_now();
9112                    }
9113                }
9114            }
9115            remove_module_cgroup(placement, &cgroup_name);
9116        }
9117        drop(roster_guard);
9118        return Err(SuperviseError::Spawn {
9119            program: spec.program.clone(),
9120            source: io::Error::other(
9121                "the daemon began shutting down while this process was starting; ended it",
9122            ),
9123            cgroup_path,
9124        });
9125    }
9126
9127    #[cfg(windows)]
9128    if let Err(error) = resume_suspended_child(pid, spec) {
9129        kill_and_reap_rejected_child(&mut child);
9130        return Err(error);
9131    }
9132
9133    let stdout_pump = match child.stdout.take() {
9134        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9135        None => {
9136            warn!(
9137                module_id = %spec.module_id,
9138                "spawned child exposed no stdout pipe; file capture will be incomplete"
9139            );
9140            None
9141        }
9142    };
9143    let stderr_pump = match child.stderr.take() {
9144        Some(stderr) => {
9145            let generation = ring
9146                .lock()
9147                .unwrap_or_else(|poisoned| poisoned.into_inner())
9148                .begin_process();
9149            Some(StderrPump {
9150                task: tokio::spawn(pump_stderr_to(
9151                    stderr,
9152                    Arc::clone(ring),
9153                    generation,
9154                    output_sink,
9155                )),
9156                generation,
9157            })
9158        }
9159        None => {
9160            // Spawning succeeded but the pipe did not materialise. Recording it as
9161            // uncaptured keeps the tail honest: the alternative is an empty tail
9162            // that reads as a module which printed nothing.
9163            ring.lock()
9164                .unwrap_or_else(|poisoned| poisoned.into_inner())
9165                .mark_not_captured("stderr pipe was not available on spawn");
9166            warn!(
9167                module_id = %spec.module_id,
9168                "spawned child exposed no stderr pipe; tail will be unavailable"
9169            );
9170            None
9171        }
9172    };
9173
9174    let exec = Arc::new(crate::activity::ExecOutcome::new());
9175    #[cfg(target_os = "macos")]
9176    let pending_exec = privacy_exec.is_some();
9177    #[cfg(not(target_os = "macos"))]
9178    let pending_exec = false;
9179    if !pending_exec {
9180        exec.finish(crate::activity::ExecState::CommittedExec);
9181    }
9182    Ok(SupervisedChild {
9183        child,
9184        exec: crate::activity::ExecOwner(exec),
9185        #[cfg(windows)]
9186        nonce_delivery,
9187        protocol: spec.protocol,
9188        #[cfg(target_os = "linux")]
9189        module_id: cgroup_name,
9190        #[cfg(target_os = "linux")]
9191        cgroup_placement: cgroup_placement.cloned(),
9192        #[cfg(windows)]
9193        job,
9194        stdout_pump,
9195        stderr_pump,
9196        stderr_ring: Arc::clone(ring),
9197        spawned_at,
9198        registration_deadline: None,
9199        registration_watch: None,
9200        spawned_at_ms,
9201        spawned_from,
9202        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9203        spawned_file_identity,
9204        process_start_time,
9205        process_identity,
9206        pid,
9207        roster_guard: Some(roster_guard),
9208        #[cfg(target_os = "macos")]
9209        privacy_exec,
9210        #[cfg(target_os = "macos")]
9211        report_ready: Arc::new(OnceLock::new()),
9212        spawn_failure: None,
9213    })
9214}
9215
9216#[cfg(test)]
9217mod spawn_record_tests {
9218    use super::*;
9219
9220    thread_local! {
9221        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9222    }
9223
9224    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9225        LAST_PID.set(child.id());
9226        #[cfg(windows)]
9227        if let Some((_, address)) = _spec
9228            .env
9229            .iter()
9230            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9231        {
9232            use std::io::{Read, Write};
9233            let mut stream = std::net::TcpStream::connect(address).unwrap();
9234            stream
9235                .write_all(&child.id().unwrap().to_ne_bytes())
9236                .unwrap();
9237            let mut release = [0];
9238            stream.read_exact(&mut release).unwrap();
9239        }
9240    }
9241
9242    #[cfg(windows)]
9243    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9244        if spec
9245            .env
9246            .iter()
9247            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9248        {
9249            Err(io::Error::other("injected module job assignment failure"))
9250        } else {
9251            Ok(())
9252        }
9253    }
9254
9255    fn spec() -> ModuleSpec {
9256        ModuleSpec {
9257            module_id: "spawn-record-worker".into(),
9258            #[cfg(unix)]
9259            program: "/bin/sleep".into(),
9260            #[cfg(windows)]
9261            program: std::env::current_exe().unwrap(),
9262            #[cfg(unix)]
9263            args: vec!["60".into()],
9264            #[cfg(windows)]
9265            args: vec![
9266                "--exact".into(),
9267                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9268                "--nocapture".into(),
9269            ],
9270            env: vec![],
9271            reserved: false,
9272            reserved_prefixes: vec![],
9273            protocol: ModuleProtocol::None,
9274            overlap: Default::default(),
9275        }
9276    }
9277
9278    #[tokio::test]
9279    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9280        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9281        let supervisor =
9282            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9283                .with_live_children_record(home.join("live-children.json"));
9284        crate::live_children::fail_next_file_sync();
9285        let result = supervisor.spawn(spec());
9286        let failed = result.is_err();
9287        if let Ok(module) = result {
9288            module.stop().await.unwrap();
9289        }
9290        assert!(failed, "record failure must fail spawn");
9291        let pid = LAST_PID.get().expect("the real OS child was created");
9292        assert!(
9293            subc_os::Process::open(pid).unwrap().is_none(),
9294            "child {pid} survived rejected admission or was not reaped"
9295        );
9296    }
9297
9298    #[cfg(target_os = "macos")]
9299    #[test]
9300    #[ignore = "manual on-device latency measurement"]
9301    fn durable_record_write_latency_50() {
9302        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9303        let path = home.join("live-children.json");
9304        let children = vec![crate::live_children::LiveChild {
9305            module_id: "latency-worker".into(),
9306            pid: std::process::id(),
9307            protocol: ModuleProtocol::None,
9308            start_time: Some(1),
9309            executable: None,
9310            cgroup_name: None,
9311        }];
9312        let mut samples = Vec::new();
9313        for _ in 0..50 {
9314            let start = std::time::Instant::now();
9315            crate::live_children::write_record(&path, &children).unwrap();
9316            samples.push(start.elapsed());
9317        }
9318        samples.sort();
9319        eprintln!(
9320            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9321            (samples[24] + samples[25]) / 2,
9322            samples[47]
9323        );
9324    }
9325
9326    #[cfg(windows)]
9327    #[test]
9328    fn windows_store_touch_fixture() {
9329        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9330            std::fs::write(path, b"module ran").unwrap();
9331        }
9332    }
9333
9334    #[cfg(windows)]
9335    #[tokio::test]
9336    async fn windows_daemon_fixture() {
9337        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9338            return;
9339        };
9340        let mut spec = spec();
9341        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9342        let supervisor =
9343            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9344        let _module = supervisor.spawn(spec).unwrap();
9345    }
9346
9347    #[cfg(windows)]
9348    #[test]
9349    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9350        use std::io::Read;
9351        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9352        // An outer job models a scheduler/service host. The daemon's root and
9353        // root job must nest without requiring breakaway from this job.
9354        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9355        command
9356            .args([
9357                "--exact",
9358                "supervise::spawn_record_tests::windows_daemon_fixture",
9359                "--nocapture",
9360            ])
9361            .env(
9362                "SUBC_TEST_DAEMON_BARRIER",
9363                listener.local_addr().unwrap().to_string(),
9364            );
9365        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9366        listener.set_nonblocking(true).unwrap();
9367        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9368        let mut stream = loop {
9369            if let Ok((stream, _)) = listener.accept() {
9370                break stream;
9371            }
9372            assert!(
9373                daemon.child.try_wait().unwrap().is_none(),
9374                "daemon failed before rendezvous"
9375            );
9376            assert!(
9377                std::time::Instant::now() < deadline,
9378                "daemon did not reach CreateProcess failpoint"
9379            );
9380            std::thread::yield_now();
9381        };
9382        stream
9383            .set_read_timeout(Some(Duration::from_secs(30)))
9384            .unwrap();
9385        let mut pid = [0; 4];
9386        stream.read_exact(&mut pid).unwrap();
9387        let pid = u32::from_ne_bytes(pid);
9388        assert!(subc_jobobject::process_exists(pid));
9389        // Terminate only the daemon, not its outer job: otherwise the host job
9390        // would also kill a child whose daemon never joined its root job.
9391        daemon.child.kill().unwrap();
9392        daemon.child.wait().unwrap();
9393        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9394        if !exited {
9395            daemon.job.terminate().unwrap();
9396        }
9397        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9398    }
9399
9400    #[cfg(windows)]
9401    #[tokio::test]
9402    async fn windows_containment_failure_never_resumes() {
9403        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9404        let sentinel = home.join("store-touched");
9405        let mut spec = spec();
9406        spec.env
9407            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9408        spec.env.push((
9409            "SUBC_TEST_STORE_TOUCH".into(),
9410            sentinel.display().to_string(),
9411        ));
9412        let supervisor =
9413            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9414        let result = supervisor.spawn(spec);
9415        // A version that resumes despite assignment failure must have time to
9416        // execute its sentinel before dropping the returned module kills it.
9417        if let Ok(module) = result {
9418            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9419                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9420                    tokio::task::yield_now().await;
9421                }
9422            })
9423            .await;
9424            let _ = module.stop().await;
9425            panic!(
9426                "assignment failure resumed a module; store touched={}",
9427                sentinel.exists()
9428            );
9429        }
9430        assert!(
9431            !sentinel.exists(),
9432            "refused module executed store-touch sentinel"
9433        );
9434        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9435    }
9436
9437    #[cfg(windows)]
9438    #[tokio::test]
9439    async fn windows_nested_daemon_fixture() {
9440        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9441            return;
9442        };
9443        let mut spec = spec();
9444        spec.env.push((
9445            "SUBC_TEST_STORE_TOUCH".into(),
9446            sentinel.to_str().unwrap().into(),
9447        ));
9448        let supervisor =
9449            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9450        let module = supervisor
9451            .spawn(spec)
9452            .expect("per-module job must nest below daemon and host jobs");
9453        tokio::time::timeout(Duration::from_secs(10), async {
9454            while !PathBuf::from(&sentinel).exists() {
9455                tokio::task::yield_now().await;
9456            }
9457        })
9458        .await
9459        .unwrap();
9460        module.stop().await.unwrap();
9461    }
9462
9463    #[cfg(windows)]
9464    #[test]
9465    fn windows_module_job_nests_under_host_job() {
9466        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9467        let sentinel = home.join("nested-store-touched");
9468        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9469        command
9470            .args([
9471                "--exact",
9472                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9473                "--nocapture",
9474            ])
9475            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9476        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9477        assert!(daemon.child.wait().unwrap().success());
9478        assert!(sentinel.exists(), "nested protocol:none module never ran");
9479    }
9480}
9481
9482#[cfg(target_os = "linux")]
9483pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9484    use subc_cgroup::KillOutcome;
9485    match subc_cgroup::kill_module(placement, module_id) {
9486        KillOutcome::Killed => {}
9487        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9488            debug!(
9489                module_id,
9490                "cgroup tree kill unavailable; using direct-child kill"
9491            );
9492        }
9493        KillOutcome::IoError { path, error } => {
9494            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9495        }
9496    }
9497}
9498
9499/// Assign a freshly spawned Windows child without ever resuming on failure.
9500///
9501/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9502/// child assigned **while it is still suspended** (step 1 is
9503/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9504/// the child's roster admission is durable. Failed containment kills and reaps
9505/// the suspended child instead of running an unprotected module.
9506#[cfg(windows)]
9507fn contain_spawned_child(
9508    child: &mut Child,
9509    spec: &ModuleSpec,
9510) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9511    let assignment = || {
9512        let job = subc_jobobject::JobObject::new()?;
9513        #[cfg(test)]
9514        spawn_record_tests::assignment_failpoint(spec)?;
9515        job.assign(child)?;
9516        Ok::<_, io::Error>(job)
9517    };
9518    match assignment() {
9519        Ok(job) => Ok(Some(job)),
9520        Err(source) => {
9521            kill_and_reap_rejected_child(child);
9522            Err(SuperviseError::Spawn {
9523                program: spec.program.clone(),
9524                source: io::Error::other(format!(
9525                    "Windows module job containment failed: {source}"
9526                )),
9527                cgroup_path: None,
9528            })
9529        }
9530    }
9531}
9532
9533/// Rejection is synchronous: do not return a spawn error while the child can
9534/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9535fn kill_and_reap_rejected_child(child: &mut Child) {
9536    #[cfg(unix)]
9537    if let Some(pid) = child
9538        .id()
9539        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9540    {
9541        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9542    }
9543    let _ = child.start_kill();
9544    while matches!(child.try_wait(), Ok(None)) {
9545        std::thread::yield_now();
9546    }
9547}
9548
9549/// Resume a suspended child, killing it if it cannot be started.
9550///
9551/// A suspended process holds a pid and does nothing, so there is no useful
9552/// state to return: the caller gets an error and the spawn fails.
9553#[cfg(windows)]
9554fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9555    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9556        // Kill it here rather than leaving a suspended process for the caller
9557        // to notice; `kill_on_drop` would eventually do this, but the module
9558        // would have been reported as running in between.
9559        let _ = std::process::Command::new("taskkill.exe")
9560            .args(["/PID", &pid.to_string(), "/T", "/F"])
9561            .stdin(Stdio::null())
9562            .stdout(Stdio::null())
9563            .stderr(Stdio::null())
9564            .status();
9565        return Err(SuperviseError::Spawn {
9566            program: spec.program.clone(),
9567            source,
9568            cgroup_path: None,
9569        });
9570    }
9571    Ok(())
9572}
9573
9574#[cfg(target_os = "linux")]
9575fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9576    match placement.remove_module(module_id) {
9577        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9578        Err(error) => warn!(
9579            module_id,
9580            error = %error,
9581            "could not remove module cgroup after process exit; continuing teardown"
9582        ),
9583    }
9584}
9585
9586#[cfg(target_os = "linux")]
9587async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9588    // Reaping the direct child is not proof its descendants exited. End the
9589    // residual tree and wait for the kernel's population fact before rmdir;
9590    // otherwise a successful parent wait leaks a directory on each restart.
9591    if matches!(
9592        subc_cgroup::kill_module(Some(placement), module_id),
9593        subc_cgroup::KillOutcome::Killed
9594    ) {
9595        if let Ok(path) = placement.module_path(module_id) {
9596            while std::fs::read_to_string(path.join("cgroup.events"))
9597                .ok()
9598                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9599            {
9600                sleep(Duration::from_millis(1)).await;
9601            }
9602        }
9603    }
9604    remove_module_cgroup(placement, module_id);
9605}
9606
9607#[cfg(target_os = "linux")]
9608fn apply_cgroup_placement(
9609    command: &mut Command,
9610    spec: &ModuleSpec,
9611    path: &std::path::Path,
9612) -> Result<(), SuperviseError> {
9613    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9614        module_id: spec.module_id.clone(),
9615        source,
9616    })
9617}
9618
9619fn capture_retention(spec: &ModuleSpec) -> Retention {
9620    let defaults = Retention::default();
9621    let value = |name: &str| {
9622        spec.env
9623            .iter()
9624            .rev()
9625            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9626    };
9627    Retention {
9628        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9629            .and_then(|value| value.parse().ok())
9630            .unwrap_or(defaults.max_file_mb),
9631        keep: value(CAPTURE_KEEP_ENV)
9632            .and_then(|value| value.parse().ok())
9633            .unwrap_or(defaults.keep),
9634        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9635            .and_then(|value| value.parse().ok())
9636            .unwrap_or(defaults.max_age_days),
9637    }
9638}
9639
9640/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9641/// module's registration to the exact process the supervisor spawned.
9642fn generate_launch_nonce() -> Result<String, SuperviseError> {
9643    let mut bytes = [0u8; 32];
9644    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9645        reason: source.to_string(),
9646    })?;
9647    let mut hex = String::with_capacity(64);
9648    for b in bytes {
9649        use std::fmt::Write;
9650        let _ = write!(hex, "{b:02x}");
9651    }
9652    Ok(hex)
9653}
9654
9655/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9656/// signal about how many leading bytes matched.
9657fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9658    if a.len() != b.len() {
9659        return false;
9660    }
9661    let mut diff = 0u8;
9662    for (x, y) in a.iter().zip(b.iter()) {
9663        diff |= x ^ y;
9664    }
9665    diff == 0
9666}
9667
9668/// The kernel's image after an acknowledged exec, shared by ordinary launches
9669/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9670/// not identities inferred from a configured pathname.
9671#[cfg(not(windows))]
9672fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9673    let process = subc_os::Process::open(pid).ok().flatten()?;
9674    #[cfg(target_os = "linux")]
9675    return sample_after_linux_exec(
9676        || {
9677            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9678            let committed = linux_exec_committed(&stat);
9679            #[cfg(test)]
9680            spawn_image_tests::report_exec_state(committed);
9681            committed
9682        },
9683        || process.observe(),
9684    );
9685    #[cfg(not(target_os = "linux"))]
9686    process.observe()
9687}
9688
9689/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9690/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9691/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9692#[cfg(target_os = "linux")]
9693fn linux_exec_committed(stat: &str) -> Option<bool> {
9694    const PF_FORKNOEXEC: u64 = 0x40;
9695    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9696    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9697}
9698
9699#[cfg(target_os = "linux")]
9700fn sample_after_linux_exec(
9701    mut exec_committed: impl FnMut() -> Option<bool>,
9702    observe: impl FnOnce() -> Option<subc_os::Observation>,
9703) -> Option<subc_os::Observation> {
9704    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9705    while !exec_committed()? {
9706        // A failed or indefinitely stalled exec must not publish the inherited
9707        // image as authority to signal a future orphan. Missing identity keeps
9708        // the existing conservative orphan-sweep behavior.
9709        if std::time::Instant::now() >= deadline {
9710            return None;
9711        }
9712        std::thread::sleep(Duration::from_micros(50));
9713    }
9714    observe()
9715}
9716
9717#[cfg(all(test, target_os = "linux"))]
9718mod spawn_image_tests {
9719    use super::*;
9720    use std::cell::{Cell, RefCell};
9721
9722    enum ExecEvent {
9723        Uncommitted,
9724        Image(Option<subc_os::Observation>),
9725    }
9726
9727    thread_local! {
9728        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9729    }
9730
9731    pub(super) fn report_exec_state(committed: Option<bool>) {
9732        if committed == Some(false) {
9733            EXEC_EVENTS.with(|events| {
9734                if let Some(events) = events.borrow_mut().take() {
9735                    let _ = events.send(ExecEvent::Uncommitted);
9736                }
9737            });
9738        }
9739    }
9740
9741    #[test]
9742    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9743        use std::io::{Read, Write};
9744        use std::os::fd::AsRawFd;
9745
9746        let descriptor = std::fs::File::open("/dev/null").unwrap();
9747        let mut command = std::process::Command::new("/bin/sleep");
9748        command.arg("30");
9749        let (mut ready, mut release) =
9750            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9751                .unwrap();
9752        let spawn = std::thread::spawn(move || command.spawn());
9753        let mut message = [0u8; 8];
9754        ready.read_exact(&mut message).unwrap();
9755        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9756        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9757        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9758        let (events, received) = std::sync::mpsc::channel();
9759        let observer = std::thread::spawn(move || {
9760            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9761            let image = observe_spawned_image(pid);
9762            let _ = events.send(ExecEvent::Image(image));
9763        });
9764        // Release only once the production observer has seen the uncommitted
9765        // exec, or has incorrectly returned the inherited image. Either way,
9766        // release and reap before asserting so a regression cannot leak a child.
9767        let first = received.recv_timeout(Duration::from_secs(5));
9768        release.write_all(&[1]).unwrap();
9769        let mut child = spawn.join().unwrap().unwrap();
9770        let after = match first {
9771            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9772                Ok(ExecEvent::Image(image)) => image,
9773                _ => None,
9774            },
9775            Ok(ExecEvent::Image(image)) => image,
9776            Err(_) => None,
9777        };
9778        observer.join().unwrap();
9779        child.kill().unwrap();
9780        child.wait().unwrap();
9781
9782        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9783        assert_eq!(
9784            inherited.unwrap().executable,
9785            subc_os::file_identity(&std::env::current_exe().unwrap())
9786        );
9787        assert_eq!(
9788            after.unwrap().executable,
9789            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9790        );
9791    }
9792
9793    #[test]
9794    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9795        let committed = Cell::new(false);
9796        let inherited = subc_os::Observation {
9797            start_time: 23,
9798            executable: Some(subc_os::FileIdentity {
9799                device: 29,
9800                inode: 31,
9801            }),
9802        };
9803        let module = subc_os::Observation {
9804            start_time: 23,
9805            executable: Some(subc_os::FileIdentity {
9806                device: 29,
9807                inode: 37,
9808            }),
9809        };
9810        let polls = Cell::new(0);
9811        let image = sample_after_linux_exec(
9812            || {
9813                polls.set(polls.get() + 1);
9814                // Hold the inherited image through the first poll, as in the
9815                // posix_spawn parent-wake window, then commit the module image.
9816                committed.set(polls.get() == 2);
9817                Some(committed.get())
9818            },
9819            || Some(if committed.get() { module } else { inherited }),
9820        );
9821        assert_eq!(image, Some(module));
9822        assert_eq!(polls.get(), 2);
9823    }
9824
9825    #[test]
9826    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9827        assert_eq!(
9828            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9829            Some(false)
9830        );
9831        assert_eq!(
9832            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9833            Some(true)
9834        );
9835        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9836        assert_eq!(linux_exec_committed("17 (a) R"), None);
9837    }
9838
9839    #[test]
9840    fn unreadable_exec_state_never_samples_an_inherited_image() {
9841        assert_eq!(
9842            sample_after_linux_exec(
9843                || None,
9844                || panic!("must not sample before exec confirmation")
9845            ),
9846            None
9847        );
9848    }
9849
9850    #[test]
9851    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9852        assert_eq!(
9853            sample_after_linux_exec(
9854                || Some(false),
9855                || panic!("must not sample after an exec timeout")
9856            ),
9857            None
9858        );
9859    }
9860}
9861
9862/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9863/// different executable after the no-delete capture was taken.
9864#[cfg(windows)]
9865fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9866    let mut program = spec.program.clone();
9867    if program.is_absolute() || program.components().count() > 1 {
9868        // An explicit path can name a PE image with no extension. Try that
9869        // exact file first, as Command does, before falling back to .exe.
9870        match subc_os::ExecutableCapture::open(&program) {
9871            Ok(capture) => return Ok(capture),
9872            Err(error)
9873                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9874            Err(error) => return Err(error),
9875        }
9876        program.set_extension("exe");
9877        return subc_os::ExecutableCapture::open(&program);
9878    }
9879    if program.extension().is_none() {
9880        program.set_extension("exe");
9881    }
9882    let custom_path = spec
9883        .env
9884        .iter()
9885        .rev()
9886        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9887    let mut directories = Vec::new();
9888    // These are Windows' default executable search locations. A configured
9889    // PATH still overrides the inherited PATH, not the system directories.
9890    if let Some(parent) = std::env::current_exe()?.parent() {
9891        directories.push(parent.to_path_buf());
9892    }
9893    directories.push(std::env::current_dir()?);
9894    if let Some(windows) = std::env::var_os("SystemRoot") {
9895        let windows = PathBuf::from(windows);
9896        directories.push(windows.join("System32"));
9897        directories.push(windows.clone());
9898    }
9899    let path = custom_path
9900        .map(|(_, value)| std::ffi::OsString::from(value))
9901        .or_else(|| std::env::var_os("PATH"));
9902    if let Some(path) = path {
9903        directories.extend(std::env::split_paths(&path));
9904    }
9905    for directory in directories {
9906        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9907            Ok(capture) => return Ok(capture),
9908            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9909            Err(error) => return Err(error),
9910        }
9911    }
9912    Err(io::Error::new(
9913        io::ErrorKind::NotFound,
9914        "resolved executable not found",
9915    ))
9916}
9917
9918fn spawn_and_mark_running(
9919    spec: &ModuleSpec,
9920    runtime: &SupervisorRuntimeConfig,
9921    snapshot: &SharedSnapshot,
9922) -> Result<SupervisedChild, SuperviseError> {
9923    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9924    let registration_watch = watch_registration(&runtime.registry, spec);
9925    let child = spawn_child(
9926        spec,
9927        runtime.connection_file_path.as_deref(),
9928        runtime.supervisor_handle.as_ref(),
9929        &runtime.stderr_ring,
9930        runtime.capture_logs_dir.as_deref(),
9931        &runtime.child_roster,
9932        #[cfg(target_os = "linux")]
9933        runtime.cgroup_placement.as_ref(),
9934    )?;
9935    let mut child = child;
9936    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
9937    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9938    if let Some(spawn) = activity_spawn {
9939        spawn.complete(snapshot);
9940    }
9941    Ok(child)
9942}
9943
9944struct RegistrationWatch {
9945    changes: watch::Receiver<u64>,
9946    baseline: u64,
9947}
9948
9949// Capture before spawning: a fast child's accepted HELLO can disappear from the
9950// active registry before the monitor first runs. A pre-existing registration
9951// must not satisfy a new process's deadline either.
9952fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
9953    if spec.protocol != ModuleProtocol::Subc {
9954        return None;
9955    }
9956    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
9957    let baseline = *changes.borrow();
9958    Some(RegistrationWatch { changes, baseline })
9959}
9960
9961fn arm_registration_deadline(
9962    child: &mut SupervisedChild,
9963    budget: Duration,
9964    registration_watch: Option<RegistrationWatch>,
9965) {
9966    child.registration_deadline =
9967        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
9968    child.registration_watch = registration_watch;
9969}
9970
9971async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
9972    #[cfg(unix)]
9973    {
9974        // Children lead their own process groups. Signal the whole group so a
9975        // helper stuck before HELLO cannot survive its supervised parent.
9976        let group = unreaped_registration_group(child)?;
9977        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
9978        // Process-group safety rule: only signal a group while its leader is
9979        // our unreaped child. Even a zombie leader reserves its pid/group id;
9980        // reaping it before KILL would let that id name an unrelated group.
9981        // Sleep without waiting so TERM-ignoring helpers get the full grace.
9982        sleep(crate::child_roster::TERM_TO_KILL).await;
9983        let group = unreaped_registration_group(child)?;
9984        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
9985    }
9986    // Windows terminates the containing Job Object as well as the direct
9987    // child. Linux also kills the child's cgroup when placement is enabled.
9988    child.start_kill()?;
9989    child.wait().await?;
9990    Ok(())
9991}
9992
9993#[cfg(unix)]
9994fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
9995    // The group id comes only from the pid recorded at spawn, when the child
9996    // created its own group. Never admit init's group or a negative group id.
9997    let group = i32::try_from(child.pid)
9998        .ok()
9999        .filter(|pid| *pid > 1)
10000        .and_then(rustix::process::Pid::from_raw)
10001        .ok_or_else(|| {
10002            io::Error::new(
10003                io::ErrorKind::InvalidInput,
10004                "invalid supervised process group",
10005            )
10006        })?;
10007    if child.child.id() != Some(child.pid) {
10008        return Err(io::Error::new(
10009            io::ErrorKind::InvalidInput,
10010            "refusing process-group signal after supervised child was reaped",
10011        ));
10012    }
10013    Ok(group)
10014}
10015
10016enum RegistrationWaitOutcome {
10017    Registered,
10018    Exited(ExitReport),
10019    TimedOut,
10020}
10021
10022struct ReloadRegistrationFailure {
10023    exit_report: ExitReport,
10024    reason: String,
10025}
10026
10027#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10028enum BusyGaugeObservation {
10029    Quiescent,
10030    Busy,
10031    Omitted,
10032}
10033
10034/// What one health answer said about one declared busy gauge.
10035#[derive(Debug, Clone, PartialEq, Eq)]
10036enum GaugeVerdict {
10037    /// The gauge read 0: this gauge is not what holds the drain.
10038    Zero,
10039    /// The gauge read a positive count.
10040    NonZero(u64),
10041    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10042    /// a negative number). It counts as busy, because the daemon cannot tell
10043    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10044    NotACount(String),
10045    /// The health answer had no metrics object, or one without this gauge.
10046    Missing,
10047    /// The probe produced no health answer to read; carries the probe error's
10048    /// evidence label (`no-answer` for a probe that timed out).
10049    ProbeFailed(&'static str),
10050}
10051
10052impl GaugeVerdict {
10053    fn reads_quiet(&self) -> bool {
10054        matches!(self, Self::Zero)
10055    }
10056}
10057
10058impl fmt::Display for GaugeVerdict {
10059    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10060        match self {
10061            Self::Zero => f.write_str("zero"),
10062            Self::NonZero(value) => write!(f, "non_zero({value})"),
10063            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10064            Self::Missing => f.write_str("missing_from_health_body"),
10065            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10066        }
10067    }
10068}
10069
10070/// The longest JSON text kept for a gauge value that is not a count, so a
10071/// module reporting a large object as a gauge cannot flood the drain log.
10072const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10073
10074/// Read each declared gauge out of one health answer's metrics.
10075fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10076    let metrics = metrics.and_then(Value::as_object);
10077    gauges
10078        .iter()
10079        .map(
10080            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10081                None => GaugeVerdict::Missing,
10082                Some(value) => match value.as_u64() {
10083                    Some(0) => GaugeVerdict::Zero,
10084                    Some(count) => GaugeVerdict::NonZero(count),
10085                    None => {
10086                        let mut text = value.to_string();
10087                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10088                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10089                            while !text.is_char_boundary(end) {
10090                                end -= 1;
10091                            }
10092                            text.truncate(end);
10093                            text.push_str("...");
10094                        }
10095                        GaugeVerdict::NotACount(text)
10096                    }
10097                },
10098            },
10099        )
10100        .collect()
10101}
10102
10103/// The drain's overall reading of one health answer: omitted when any
10104/// declared gauge is missing, quiescent only when every gauge reads 0.
10105fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10106    if verdicts
10107        .iter()
10108        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10109    {
10110        BusyGaugeObservation::Omitted
10111    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10112        BusyGaugeObservation::Quiescent
10113    } else {
10114        BusyGaugeObservation::Busy
10115    }
10116}
10117
10118/// One declared gauge's history over a drain: the latest verdict on it, and
10119/// the latest value actually read for it (which a later failed probe or
10120/// missing gauge does not erase). Times are offsets from the start of the
10121/// drain's wait.
10122#[derive(Debug, Clone, Default, PartialEq, Eq)]
10123struct GaugeTrace {
10124    latest: Option<(GaugeVerdict, Duration)>,
10125    last_value: Option<(GaugeVerdict, Duration)>,
10126}
10127
10128/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10129/// its ceiling can say in its timeout line which declared gauge held it and
10130/// why. Without it, a gauge that read busy, a gauge the module never
10131/// reported, and a probe that never got an answer all looked the same: a
10132/// drain that ended at its ceiling with nothing in flight.
10133#[derive(Debug, Clone, Default, PartialEq, Eq)]
10134struct DrainGaugeTrace {
10135    /// Health probes the drain sent.
10136    probes: u32,
10137    /// Each declared gauge, in declaration order.
10138    gauges: Vec<(String, GaugeTrace)>,
10139}
10140
10141impl DrainGaugeTrace {
10142    fn new(gauges: &[String]) -> Self {
10143        Self {
10144            probes: 0,
10145            gauges: gauges
10146                .iter()
10147                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10148                .collect(),
10149        }
10150    }
10151
10152    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10153    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10154        self.probes = self.probes.saturating_add(1);
10155        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10156            trace.latest = Some((verdict.clone(), at));
10157            if matches!(
10158                verdict,
10159                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10160            ) {
10161                trace.last_value = Some((verdict.clone(), at));
10162            }
10163        }
10164    }
10165}
10166
10167/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10168/// declared gauge, separated by ` | `. A value is shown as its verdict
10169/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10170/// and whether that counted as busy.
10171impl fmt::Display for DrainGaugeTrace {
10172    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10173        if self.gauges.is_empty() {
10174            return f.write_str("none declared");
10175        }
10176        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10177            if index > 0 {
10178                f.write_str(" | ")?;
10179            }
10180            write!(f, "{name}: ")?;
10181            match &trace.latest {
10182                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10183                None => f.write_str("never probed")?,
10184            }
10185            match &trace.last_value {
10186                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10187                None => f.write_str("; no value ever read")?,
10188            }
10189        }
10190        Ok(())
10191    }
10192}
10193
10194pub(crate) fn declared_busy_gauges(
10195    registry: &Registry,
10196    module_id: &str,
10197) -> Result<Vec<String>, SuperviseError> {
10198    busy_gauges_of(
10199        registry
10200            .get_module(module_id)
10201            .map_err(SuperviseError::Registry)?,
10202    )
10203}
10204
10205/// [`declared_busy_gauges`] for the registration a connection holds, in any
10206/// slot: after cutover the incumbent is no longer the id's active
10207/// registration, and its own manifest is the one that names its gauges.
10208fn declared_busy_gauges_for_connection(
10209    registry: &Registry,
10210    connection_id: ConnectionId,
10211) -> Result<Vec<String>, SuperviseError> {
10212    busy_gauges_of(
10213        registry
10214            .get_module_by_connection(connection_id)
10215            .map_err(SuperviseError::Registry)?,
10216    )
10217}
10218
10219fn busy_gauges_of(
10220    registration: Option<crate::registry::ModuleRegistration>,
10221) -> Result<Vec<String>, SuperviseError> {
10222    let Some(registration) = registration else {
10223        return Ok(Vec::new());
10224    };
10225    let Some(self_signals) = registration.manifest.self_signals else {
10226        return Ok(Vec::new());
10227    };
10228
10229    let mut gauges = Vec::new();
10230    for declaration in self_signals {
10231        if declaration.kind != SelfSignalKind::Busy {
10232            continue;
10233        }
10234        match declaration.anchored_to {
10235            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10236                gauges.extend(declared)
10237            }
10238            _ => {
10239                // An invalid Busy anchor is fail-safe: the empty name cannot be
10240                // present in a conforming health report, so this drain stays busy.
10241                gauges.push(String::new());
10242            }
10243        }
10244    }
10245    Ok(gauges)
10246}
10247
10248/// Wait for `endpoint` to have nothing in flight and, when the module declares
10249/// busy gauges, for a health probe to report them quiet. The probe is addressed
10250/// by `scope`: a swap's superseded incumbent must be asked about its own
10251/// gauges, and by module id the probe would reach the promoted candidate.
10252/// While the gauges read busy, the probe repeats every
10253/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10254/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10255/// timeout line.
10256#[allow(clippy::too_many_arguments)]
10257async fn wait_for_forwarding_quiescence(
10258    forwarding: &ForwardingTable,
10259    module_id: &str,
10260    runtime: &SupervisorRuntimeConfig,
10261    endpoint: crate::ModuleEndpointId,
10262    deadline: Instant,
10263    busy_gauges: &[String],
10264    scope: DrainScope,
10265    trace: &mut DrainGaugeTrace,
10266) -> Result<bool, SuperviseError> {
10267    let wait_started = Instant::now();
10268    let mut gauges_quiescent = busy_gauges.is_empty();
10269    let mut next_probe_at = wait_started;
10270    let mut omission_reported = false;
10271
10272    loop {
10273        let now = Instant::now();
10274        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10275            let report = match scope {
10276                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10277                DrainScope::Endpoint(endpoint) => {
10278                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10279                }
10280            };
10281            let verdicts = match &report {
10282                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10283                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10284            };
10285            trace.record(&verdicts, wait_started.elapsed());
10286            gauges_quiescent = match report {
10287                Ok(_) => match busy_gauge_observation(&verdicts) {
10288                    BusyGaugeObservation::Quiescent => true,
10289                    BusyGaugeObservation::Busy => false,
10290                    BusyGaugeObservation::Omitted => {
10291                        // Once per drain: a module that never reports a gauge
10292                        // it declared would otherwise say so on every probe.
10293                        if !omission_reported {
10294                            forwarding
10295                                .counters()
10296                                .increment_drains_with_undeclared_gauge();
10297                            let missing = busy_gauges
10298                                .iter()
10299                                .zip(&verdicts)
10300                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10301                                .map(|(gauge, _)| gauge.as_str())
10302                                .collect::<Vec<_>>()
10303                                .join(",");
10304                            warn!(
10305                                module_id,
10306                                missing_gauges = %missing,
10307                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10308                            );
10309                            omission_reported = true;
10310                        }
10311                        false
10312                    }
10313                },
10314                Err(err) => {
10315                    warn!(
10316                        module_id,
10317                        error = %err,
10318                        "drain health.check did not produce declared busy gauges; treating module as busy"
10319                    );
10320                    false
10321                }
10322            };
10323            // Counted from the probe's completion: the probe above was awaited,
10324            // so no second probe can start while one is still outstanding.
10325            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10326        }
10327
10328        let in_flight = forwarding
10329            .endpoint_in_flight_count(endpoint)
10330            .map_err(SuperviseError::Forwarding)?;
10331        if in_flight == 0 && gauges_quiescent {
10332            return Ok(true);
10333        }
10334
10335        let now = Instant::now();
10336        if now >= deadline {
10337            return Ok(false);
10338        }
10339        let mut wait = deadline
10340            .saturating_duration_since(now)
10341            .min(REGISTRY_RELEASE_POLL);
10342        if !busy_gauges.is_empty() {
10343            wait = wait.min(next_probe_at.saturating_duration_since(now));
10344        }
10345        sleep(wait).await;
10346    }
10347}
10348
10349/// The wait between drain busy-gauge probes: the larger of
10350/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10351/// drain's own in-flight poll interval.
10352fn drain_busy_reprobe_interval() -> Duration {
10353    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10354}
10355
10356/// Logs why a drain gave up: the drain waited its full timeout without the
10357/// module going quiet, and teardown is being forced.
10358///
10359/// The line names two kinds of evidence, because a bare "timed out" hides
10360/// causes that need different fixes:
10361/// - `holdouts`: the requests and routes still open when the timeout hit,
10362///   and on which connections. One stuck call is a module bug; a held stream
10363///   on every session is a module that should end its streams when it
10364///   receives `module.draining`.
10365/// - `gauges`: for each busy gauge the module declares, its latest reading
10366///   and how many probes ran. This separates a module that kept reporting
10367///   itself busy, one whose health answer never contained the gauge, and one
10368///   that never answered the probes, which otherwise all look like a timeout
10369///   with nothing in flight.
10370fn warn_drain_timed_out(
10371    module_id: &str,
10372    drain_timeout: Duration,
10373    reason: RouteCloseReason,
10374    holdouts: &crate::forwarding::DrainHoldouts,
10375    gauges: &DrainGaugeTrace,
10376) {
10377    warn!(
10378        module_id = %module_id,
10379        waited = ?drain_timeout,
10380        ?reason,
10381        held_requests = holdouts.requests,
10382        held_routes = holdouts.routes,
10383        total_routes = holdouts.total_routes,
10384        top_connections = ?holdouts.top_connections,
10385        // `module_channel:corr`, so the module can find each held request
10386        // in its own log; capped, so `held_requests` is the full count.
10387        held = %holdouts
10388            .held
10389            .iter()
10390            .map(|(channel, corr)| format!("{channel}:{corr}"))
10391            .collect::<Vec<_>>()
10392            .join(","),
10393        gauge_probes = gauges.probes,
10394        busy_gauges = %gauges,
10395        "route drain timed out before request quiescence; forcing teardown"
10396    );
10397}
10398
10399/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10400///
10401/// `Ok` is always honest and passed straight through -- the wait actually measured
10402/// in-flight state. `Err` means the wait produced no measurement at all (the
10403/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10404/// constant: the drain did not complete. Never recomputed from route state, never a
10405/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10406fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10407    match wait_result {
10408        Ok(drained) => *drained,
10409        Err(_) => false,
10410    }
10411}
10412
10413fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10414    for released in released_routes {
10415        let frame = match Frame::build_with_version(
10416            released.negotiated_ver,
10417            FrameType::Goodbye,
10418            control_flags(),
10419            released.channel,
10420            released.epoch,
10421            0,
10422            Vec::new(),
10423        ) {
10424            Ok(frame) => frame,
10425            Err(err) => {
10426                warn!(
10427                    route_channel = released.channel,
10428                    error = %err,
10429                    "failed to build supervisor drain route GOODBYE frame"
10430                );
10431                continue;
10432            }
10433        };
10434        if !released.close_on_delivery_failure() {
10435            crate::forwarding::send_module_route_goodbye(
10436                &forwarding.counters(),
10437                &released.sink,
10438                frame,
10439                released.module_id.as_deref(),
10440                "supervisor drain",
10441            );
10442            continue;
10443        }
10444        if let Err(err) = released.sink.try_send(frame) {
10445            warn!(
10446                target_connection_id = released.connection_id.get(),
10447                route_channel = released.channel,
10448                error = %err,
10449                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10450            );
10451            let _ = forwarding.escalate_client_delivery_failure(
10452                released.connection_id,
10453                released.channel,
10454                released.epoch,
10455                CloseReason::new(
10456                    "route_goodbye_delivery_failed",
10457                    format!(
10458                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10459                        released.channel
10460                    ),
10461                ),
10462                crate::forwarding::UndeliveredFrame {
10463                    module_id: released.module_id.as_deref(),
10464                    sink: &released.sink,
10465                },
10466            );
10467        }
10468    }
10469}
10470
10471fn send_module_draining(
10472    module_id: &str,
10473    reason: RouteCloseReason,
10474    deadline_ms: u64,
10475    target: &ModuleDrainTarget,
10476) {
10477    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10478        reason,
10479        deadline_ms,
10480    }) {
10481        Ok(body) => body,
10482        Err(err) => {
10483            warn!(
10484                module_id,
10485                error = %err,
10486                "failed to encode module draining command"
10487            );
10488            return;
10489        }
10490    };
10491    let frame = match Frame::build_with_version(
10492        target.negotiated_ver,
10493        FrameType::Push,
10494        control_flags(),
10495        0,
10496        0,
10497        0,
10498        body,
10499    ) {
10500        Ok(frame) => frame,
10501        Err(err) => {
10502            warn!(
10503                module_id,
10504                error = %err,
10505                "failed to build module draining command frame"
10506            );
10507            return;
10508        }
10509    };
10510    if let Err(err) = target.sink.try_send(frame) {
10511        warn!(
10512            module_id,
10513            target_connection_id = target.endpoint.connection_id.get(),
10514            error = %err,
10515            "module draining command was not delivered to peer"
10516        );
10517    }
10518}
10519
10520/// The channel-0 GOODBYE that tells a module its stop is planned.
10521fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10522    match Frame::build_with_version(
10523        negotiated_ver,
10524        FrameType::Goodbye,
10525        control_flags(),
10526        0,
10527        0,
10528        0,
10529        Vec::new(),
10530    ) {
10531        Ok(frame) => Some(frame),
10532        Err(err) => {
10533            warn!(
10534                module_id,
10535                error = %err,
10536                "failed to build module GOODBYE frame"
10537            );
10538            None
10539        }
10540    }
10541}
10542
10543/// Send every registered module connection its module GOODBYE at daemon
10544/// shutdown, then request that connection's close.
10545///
10546/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10547/// before EOF, so the GOODBYE must reach the socket before the close. A close
10548/// request does not wait for the connection's queued frames: its writer gets a
10549/// bounded grace after the close, is aborted if it overruns it, and the daemon
10550/// process may exit before that grace ends. So with `wait_for_flush`, each
10551/// connection is closed only after its writer has acknowledged writing the
10552/// GOODBYE, or once a short shared budget runs out, so one module that is not
10553/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10554/// are only queued, for a shutdown the operator has told to stop waiting.
10555/// A connection that is already gone is skipped.
10556#[cfg(unix)]
10557async fn send_module_goodbyes_for_daemon_shutdown(
10558    forwarding: &Arc<ForwardingTable>,
10559    reason: &CloseReason,
10560    wait_for_flush: bool,
10561) {
10562    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10563    let targets = match forwarding.module_connections() {
10564        Ok(targets) => targets,
10565        Err(err) => {
10566            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10567            return;
10568        }
10569    };
10570    let deadline = Instant::now() + GOODBYE_BUDGET;
10571    let mut sends = tokio::task::JoinSet::new();
10572    for target in targets {
10573        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10574            continue;
10575        };
10576        if !wait_for_flush {
10577            if let Err(err) = target.sink.try_send(frame) {
10578                debug!(
10579                    module_id = %target.module_id,
10580                    error = %err,
10581                    "shutdown module GOODBYE was not queued"
10582                );
10583            }
10584            continue;
10585        }
10586        let forwarding = Arc::clone(forwarding);
10587        let reason = reason.clone();
10588        sends.spawn(async move {
10589            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10590                Ok(Ok(())) => {}
10591                Ok(Err(err)) => debug!(
10592                    module_id = %target.module_id,
10593                    error = %err,
10594                    "module connection closed before its shutdown GOODBYE was written"
10595                ),
10596                Err(_) => warn!(
10597                    module_id = %target.module_id,
10598                    budget = ?GOODBYE_BUDGET,
10599                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10600                ),
10601            }
10602            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10603        });
10604    }
10605    // Every task ends by the shared deadline, so this wait is bounded too.
10606    while sends.join_next().await.is_some() {}
10607}
10608
10609fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10610    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10611        return;
10612    };
10613    if let Err(err) = target.sink.try_send(frame) {
10614        warn!(
10615            module_id,
10616            target_connection_id = target.endpoint.connection_id.get(),
10617            error = %err,
10618            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10619        );
10620        forwarding.request_connection_close(
10621            target.endpoint.connection_id,
10622            CloseReason::new(
10623                "module_goodbye_delivery_failed",
10624                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10625            ),
10626        );
10627    }
10628}
10629
10630#[derive(Clone, Copy)]
10631struct ForwardingDrainContext<'a> {
10632    spec: &'a ModuleSpec,
10633    runtime: &'a SupervisorRuntimeConfig,
10634    registry: &'a Registry,
10635    scope: DrainScope,
10636}
10637
10638/// Which process a forwarding drain addresses.
10639#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10640enum DrainScope {
10641    /// Whatever endpoint is active for the module id: every plain stop,
10642    /// restart and reload. Also moves the module's state to `Draining`.
10643    Active,
10644    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10645    /// module id would resolve to the promoted candidate and leave neither
10646    /// process routable. The module's state is left alone, since the promoted
10647    /// candidate is what it describes and that process is running.
10648    Endpoint(crate::ModuleEndpointId),
10649}
10650
10651/// Whether a child being drained has already been asked to stop by the time
10652/// its drain wait starts.
10653///
10654/// The drain wait is the same budget whatever this says. What it decides is
10655/// whether the supervisor must ask by signal before that wait begins: a child
10656/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10657/// healthy or not.
10658#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10659enum StopNotice {
10660    /// The module was sent `module.draining` and a module GOODBYE over its own
10661    /// registered connection, and stops itself.
10662    SentOverConnection,
10663    /// The forwarding drain found no registered connection for the module: a
10664    /// subc child spawned moments ago that has not sent HELLO yet, or a
10665    /// `protocol: "none"` child, which never registers.
10666    NoConnection,
10667    /// This path sends nothing over the module's connection: the supervisor has
10668    /// no forwarding table, or the caller stops the child without a forwarding
10669    /// drain.
10670    NotSent,
10671}
10672
10673async fn begin_forwarding_drain(
10674    spec: &ModuleSpec,
10675    runtime: &SupervisorRuntimeConfig,
10676    registry: &Registry,
10677    snapshot: &SharedSnapshot,
10678    enabled: Option<bool>,
10679    reason: RouteCloseReason,
10680) -> Result<StopNotice, SuperviseError> {
10681    let Some(forwarding) = runtime.forwarding.as_ref() else {
10682        return Err(SuperviseError::ReloadUnavailable {
10683            module_id: spec.module_id.clone(),
10684            reason: "supervisor was not configured with a forwarding table".to_string(),
10685        });
10686    };
10687
10688    begin_forwarding_drain_with(
10689        forwarding,
10690        ForwardingDrainContext {
10691            spec,
10692            runtime,
10693            registry,
10694            scope: DrainScope::Active,
10695        },
10696        snapshot,
10697        enabled,
10698        reason,
10699        runtime.drain_timeout,
10700    )
10701    .await
10702}
10703
10704async fn begin_forwarding_drain_if_configured(
10705    spec: &ModuleSpec,
10706    runtime: &SupervisorRuntimeConfig,
10707    registry: &Registry,
10708    snapshot: &SharedSnapshot,
10709    enabled: Option<bool>,
10710    reason: RouteCloseReason,
10711) -> Result<StopNotice, SuperviseError> {
10712    begin_forwarding_drain_with_timeout(
10713        spec,
10714        runtime,
10715        registry,
10716        snapshot,
10717        enabled,
10718        reason,
10719        runtime.drain_timeout,
10720    )
10721    .await
10722}
10723
10724/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10725/// budget, for paths where the operator overrides the module's configured one
10726/// (`supervisor.restart{drain_timeout_ms}`).
10727async fn begin_forwarding_drain_with_timeout(
10728    spec: &ModuleSpec,
10729    runtime: &SupervisorRuntimeConfig,
10730    registry: &Registry,
10731    snapshot: &SharedSnapshot,
10732    enabled: Option<bool>,
10733    reason: RouteCloseReason,
10734    drain_timeout: Duration,
10735) -> Result<StopNotice, SuperviseError> {
10736    let Some(forwarding) = runtime.forwarding.as_ref() else {
10737        return Ok(StopNotice::NotSent);
10738    };
10739
10740    begin_forwarding_drain_with(
10741        forwarding,
10742        ForwardingDrainContext {
10743            spec,
10744            runtime,
10745            registry,
10746            scope: DrainScope::Active,
10747        },
10748        snapshot,
10749        enabled,
10750        reason,
10751        drain_timeout,
10752    )
10753    .await
10754}
10755
10756async fn begin_forwarding_drain_with(
10757    forwarding: &ForwardingTable,
10758    context: ForwardingDrainContext<'_>,
10759    snapshot: &SharedSnapshot,
10760    enabled: Option<bool>,
10761    reason: RouteCloseReason,
10762    drain_timeout: Duration,
10763) -> Result<StopNotice, SuperviseError> {
10764    let ForwardingDrainContext {
10765        spec,
10766        runtime,
10767        registry,
10768        scope,
10769    } = context;
10770    debug_assert_ne!(reason, RouteCloseReason::Crash);
10771    let terminal = matches!(reason, RouteCloseReason::Disable);
10772    let drain_started_at = Instant::now();
10773    let drain_deadline = drain_started_at + drain_timeout;
10774    let deadline_ms =
10775        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10776    let busy_gauges = match scope {
10777        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10778        DrainScope::Endpoint(endpoint) => {
10779            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10780        }
10781    };
10782
10783    // Admission gate first: route.open/commit and route REQUEST admission are closed
10784    // before the first quiescence check, so the outstanding count can only fall.
10785    let gate_started = Instant::now();
10786    let drain_target = match scope {
10787        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10788        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10789    }
10790    .map_err(SuperviseError::Forwarding)?;
10791    // The instant admission closed, and how long taking the forwarding write
10792    // lock to close it took. The timeout line reports only the quiescence
10793    // wait, so without this a drain that started late looked like one that
10794    // started on time.
10795    info!(
10796        module_id = %spec.module_id,
10797        ?reason,
10798        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10799        connected = drain_target.is_some(),
10800        "module drain began; route admission closed"
10801    );
10802    if scope == DrainScope::Active {
10803        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10804            state.state = ModuleState::Draining;
10805            state.draining_to_replace =
10806                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10807            if let Some(enabled) = enabled {
10808                state.enabled = enabled;
10809            }
10810        })?;
10811    }
10812
10813    let Some(target) = drain_target.as_ref() else {
10814        // Nothing was sent: the module has no registered connection to carry
10815        // `module.draining` or a GOODBYE. The caller must not assume the child
10816        // was asked to stop.
10817        return Ok(StopNotice::NoConnection);
10818    };
10819    {
10820        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10821        #[cfg(feature = "test-support")]
10822        forwarding.wait_before_route_closing_for_test().await;
10823        let routes = forwarding
10824            .endpoint_routes(target.endpoint)
10825            .map_err(SuperviseError::Forwarding)?;
10826        let routes_notified = routes.len();
10827        crate::control::send_route_control_pushes(
10828            forwarding,
10829            routes.clone(),
10830            ClientControlPush::RouteClosing {
10831                module_id: spec.module_id.clone(),
10832                channels: Vec::new(),
10833                reason,
10834            },
10835        );
10836        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10837
10838        // `route.closing` was just sent above: from here on every return path,
10839        // including an early one, MUST send `route.closed` before propagating
10840        // anything else. A client holds `closing` as a promise that a verdict is
10841        // coming; leaving early without `closed` strands it waiting forever, since
10842        // `closing` carries no timeout of its own.
10843        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10844        let wait_result = wait_for_forwarding_quiescence(
10845            forwarding,
10846            &spec.module_id,
10847            runtime,
10848            target.endpoint,
10849            drain_deadline,
10850            &busy_gauges,
10851            scope,
10852            &mut gauge_trace,
10853        )
10854        .await;
10855        let drained = drained_after_quiescence_wait(&wait_result);
10856        if let Err(err) = &wait_result {
10857            error!(
10858                module_id = %spec.module_id,
10859                ?reason,
10860                error = %err,
10861                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10862            );
10863        } else if !drained {
10864            // Read before teardown releases the routes.
10865            let holdouts = forwarding
10866                .endpoint_drain_holdouts(target.endpoint)
10867                .unwrap_or_default();
10868            warn_drain_timed_out(
10869                &spec.module_id,
10870                drain_timeout,
10871                reason,
10872                &holdouts,
10873                &gauge_trace,
10874            );
10875        }
10876        crate::control::send_route_control_pushes(
10877            forwarding,
10878            routes,
10879            ClientControlPush::RouteClosed {
10880                module_id: spec.module_id.clone(),
10881                channels: Vec::new(),
10882                reason,
10883                drained,
10884                abandoned: target.abandoned_bindings.len() as u32,
10885                excluded_subscriptions: target.excluded_subscriptions,
10886                terminal: Some(terminal),
10887            },
10888        );
10889        wait_result?;
10890
10891        // `route.closed` has now been sent unconditionally above. From here the
10892        // remaining steps are cleanup (route + module GOODBYE) rather than a
10893        // promise the client is waiting on, but a lock-poisoned
10894        // `release_module_endpoint_routes` would otherwise skip the module
10895        // GOODBYE silently too -- send it before propagating the error.
10896        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10897            Ok(routes) => routes,
10898            Err(err) => {
10899                warn!(
10900                    module_id = %spec.module_id,
10901                    ?reason,
10902                    error = %err,
10903                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10904                );
10905                send_module_goodbye(&spec.module_id, forwarding, target);
10906                return Err(SuperviseError::Forwarding(err));
10907            }
10908        };
10909        let route_goodbye_count = released_routes.len();
10910        send_route_goodbyes(forwarding, released_routes);
10911        send_module_goodbye(&spec.module_id, forwarding, target);
10912
10913        // The drain's happy path was previously silent: every emission above is
10914        // best-effort with only its failure arm logged, so "were consumers told"
10915        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10916        // hang where the open question was exactly whether teardown notice went
10917        // out). One summary line makes that class decidable in one grep.
10918        info!(
10919            module_id = %spec.module_id,
10920            ?reason,
10921            routes_notified,
10922            route_goodbyes = route_goodbye_count,
10923            abandoned_reservations = target.abandoned_bindings.len(),
10924            excluded_subscriptions = target.excluded_subscriptions,
10925            drained,
10926            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10927        );
10928    }
10929
10930    Ok(StopNotice::SentOverConnection)
10931}
10932
10933/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10934/// the only slot a plain (non-swap) spawn can register into.
10935async fn wait_for_registration_after_reload(
10936    registry: &Registry,
10937    module_id: &str,
10938    snapshot: &SharedSnapshot,
10939    child: &mut SupervisedChild,
10940    wait: Duration,
10941) -> Result<RegistrationWaitOutcome, SuperviseError> {
10942    wait_for_slot_registration(
10943        registry,
10944        crate::registry::RegistrationSlot::Active(module_id),
10945        module_id,
10946        snapshot,
10947        child,
10948        wait,
10949    )
10950    .await
10951}
10952
10953/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10954///
10955/// Keyed on the slot rather than the bare module id because during a swap the
10956/// id's active slot is already held by the incumbent: an id-keyed wait would
10957/// report the incumbent's registration as the candidate's and a candidate that
10958/// never registers would look registered. A swap candidate waits on
10959/// `crate::registry::RegistrationSlot::Candidate`.
10960async fn wait_for_slot_registration(
10961    registry: &Registry,
10962    slot: crate::registry::RegistrationSlot<'_>,
10963    module_id: &str,
10964    snapshot: &SharedSnapshot,
10965    child: &mut SupervisedChild,
10966    wait: Duration,
10967) -> Result<RegistrationWaitOutcome, SuperviseError> {
10968    let deadline = Instant::now() + wait;
10969    loop {
10970        if registry
10971            .registration(slot)
10972            .map_err(SuperviseError::Registry)?
10973            .is_some()
10974        {
10975            return Ok(RegistrationWaitOutcome::Registered);
10976        }
10977
10978        let now = Instant::now();
10979        if now >= deadline {
10980            return Ok(RegistrationWaitOutcome::TimedOut);
10981        }
10982        let remaining = deadline.saturating_duration_since(now);
10983        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10984
10985        tokio::select! {
10986            wait_result = child.wait() => {
10987                let status = wait_result.map_err(|source| SuperviseError::Wait {
10988                    module_id: module_id.to_string(),
10989                    source,
10990                })?;
10991                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10992                    snapshot,
10993                    child,
10994                    &status,
10995                )));
10996            }
10997            _ = sleep(poll) => {}
10998        }
10999    }
11000}
11001
11002fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11003    // A process stopped for missing its first HELLO did not provide service,
11004    // even if its TERM handler exits zero. Reload also uses this classification
11005    // when a replacement exits before registering.
11006    if exit_report.kind != ExitKind::DeliberateSeverance {
11007        exit_report.kind = ExitKind::Crash;
11008    }
11009    exit_report
11010}
11011
11012async fn handle_reload_child_registration_failure(
11013    spec: &ModuleSpec,
11014    runtime: &SupervisorRuntimeConfig,
11015    registry: &Registry,
11016    process_liveness: &SupervisorProcessLiveness,
11017    snapshot: &SharedSnapshot,
11018    _child: &mut Option<SupervisedChild>,
11019    failure: ReloadRegistrationFailure,
11020) -> Result<(), SuperviseError> {
11021    let ReloadRegistrationFailure {
11022        exit_report,
11023        reason,
11024    } = failure;
11025    match on_child_exit(
11026        spec,
11027        runtime.restart_policy,
11028        registry,
11029        snapshot,
11030        &runtime.terminal_ring,
11031        &runtime.spawn_events,
11032        &runtime.child_roster,
11033        exit_report,
11034        runtime.recovery_is_daemon,
11035    )
11036    .await
11037    {
11038        NextAction::Stop {
11039            registration_released,
11040        } => {
11041            if registration_released {
11042                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11043            }
11044        }
11045        NextAction::Restart { schedule } => {
11046            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11047                schedule.delay
11048            });
11049            if let Some(schedule) = schedule {
11050                log_crash_respawn(&spec.module_id, schedule);
11051            }
11052            schedule_respawn(
11053                runtime,
11054                snapshot,
11055                &spec.module_id,
11056                delay,
11057                RespawnKind::Spawn,
11058                true,
11059            )?;
11060        }
11061    }
11062    Err(SuperviseError::ReloadFailed {
11063        module_id: spec.module_id.clone(),
11064        reason,
11065    })
11066}
11067
11068async fn handle_reload_spawn_failure(
11069    spec: &ModuleSpec,
11070    runtime: &SupervisorRuntimeConfig,
11071    process_liveness: &SupervisorProcessLiveness,
11072    snapshot: &SharedSnapshot,
11073    _child: &mut Option<SupervisedChild>,
11074    reason: String,
11075) -> Result<(), SuperviseError> {
11076    let now = Instant::now();
11077    let mut schedule = None;
11078    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11079        clear_current_process_facts(state);
11080        if state.enabled {
11081            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11082            state.state = if schedule.is_some() {
11083                ModuleState::Restarting
11084            } else {
11085                if runtime
11086                    .failure_context(runtime.recovery_is_daemon)
11087                    .may_retry()
11088                {
11089                    state.failed_retry.schedule(&spec.module_id);
11090                }
11091                ModuleState::Failed
11092            };
11093        } else {
11094            state.state = ModuleState::Disabled;
11095        }
11096    })?;
11097    if let Some(schedule) = schedule {
11098        schedule_respawn(
11099            runtime,
11100            snapshot,
11101            &spec.module_id,
11102            schedule.delay,
11103            RespawnKind::Spawn,
11104            true,
11105        )?;
11106    } else {
11107        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11108    }
11109    Err(SuperviseError::ReloadFailed {
11110        module_id: spec.module_id.clone(),
11111        reason,
11112    })
11113}
11114
11115fn control_flags() -> Flags {
11116    Flags::new(false, Priority::Passive, false)
11117}
11118
11119#[allow(clippy::too_many_arguments)]
11120async fn drain_optional_child(
11121    module_id: &str,
11122    protocol: ModuleProtocol,
11123    stop_notice: StopNotice,
11124    registry: &Registry,
11125    forwarding: Option<&ForwardingTable>,
11126    snapshot: &SharedSnapshot,
11127    terminal_ring: &Arc<Mutex<TerminalRing>>,
11128    spawn_events: &SpawnEventFeed,
11129    child: &mut Option<SupervisedChild>,
11130    drain_timeout: Duration,
11131    final_state: ModuleState,
11132    enabled: Option<bool>,
11133    failure: &FailureContext,
11134) -> Result<(), SuperviseError> {
11135    if let Some(child) = child.take() {
11136        drain_child_to_state(
11137            module_id,
11138            protocol,
11139            stop_notice,
11140            registry,
11141            forwarding,
11142            snapshot,
11143            terminal_ring,
11144            spawn_events,
11145            child,
11146            drain_timeout,
11147            final_state,
11148            enabled,
11149            failure,
11150        )
11151        .await
11152    } else {
11153        update_snapshot(snapshot, Some(module_id), |state| {
11154            state.state = final_state;
11155            if final_state == ModuleState::Failed {
11156                state.failed_retry.failed_at = Some(Instant::now());
11157            }
11158            if let Some(enabled) = enabled {
11159                state.enabled = enabled;
11160            }
11161            clear_current_process_facts(state);
11162        })?;
11163        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11164    }
11165}
11166
11167#[allow(clippy::too_many_arguments)]
11168async fn drain_child_to_state(
11169    module_id: &str,
11170    _protocol: ModuleProtocol,
11171    stop_notice: StopNotice,
11172    registry: &Registry,
11173    forwarding: Option<&ForwardingTable>,
11174    snapshot: &SharedSnapshot,
11175    terminal_ring: &Arc<Mutex<TerminalRing>>,
11176    spawn_events: &SpawnEventFeed,
11177    mut child: SupervisedChild,
11178    drain_timeout: Duration,
11179    final_state: ModuleState,
11180    enabled: Option<bool>,
11181    failure: &FailureContext,
11182) -> Result<(), SuperviseError> {
11183    let protocol = child.protocol;
11184    update_snapshot(snapshot, Some(module_id), |state| {
11185        state.state = ModuleState::Draining;
11186        state.draining_to_replace = final_state == ModuleState::Restarting;
11187        if let Some(enabled) = enabled {
11188            state.enabled = enabled;
11189        }
11190    })?;
11191
11192    // The wait below is the same budget in every case; what differs is
11193    // whether anything has ASKED the child to stop before it starts. Only a
11194    // forwarding drain that reached the module's registered connection has
11195    // (`module.draining`, then a module GOODBYE). Every other child was told
11196    // nothing: a `protocol: "none"` module, which never registers; a subc
11197    // module spawned moments ago that has not sent HELLO yet; or a stop that
11198    // runs no forwarding drain. Without a signal the budget is only a delay
11199    // in front of SIGKILL -- and the not-yet-registered child is the worst
11200    // case, because it registers into a module that is already draining,
11201    // is never told, and is killed while healthy.
11202    if stop_notice != StopNotice::SentOverConnection {
11203        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11204            info!(
11205                module_id,
11206                pid = child.pid,
11207                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11208                "module has no connection yet; requesting stop by signal"
11209            );
11210        }
11211        request_graceful_stop(module_id, &child);
11212    }
11213
11214    let exit_report = match timeout(drain_timeout, child.wait()).await {
11215        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11216        Ok(Err(source)) => {
11217            fail_after_wait_error(
11218                module_id,
11219                snapshot,
11220                terminal_ring,
11221                spawn_events,
11222                format!("failed to wait during drain: {source}"),
11223                failure,
11224            );
11225            return Err(SuperviseError::Wait {
11226                module_id: module_id.to_string(),
11227                source,
11228            });
11229        }
11230        Err(_) => {
11231            // Mirror the sibling arm above: state is already `Draining`, and an
11232            // error propagated from here would strand it there -- a state
11233            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11234            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11235            // `Failed` before `?` keeps the module operator-visible and
11236            // revivable. Trigger is an ESRCH race (process exits between the
11237            // drain timeout firing and the kill) or a post-kill wait failure
11238            // (issue #34).
11239            //
11240            // Logged because the kill is otherwise visible only as signal 9 in
11241            // the terminal ring, and the budget it follows can be long enough
11242            // that consumers see a stretch of refusals with no stated cause.
11243            warn!(
11244                module_id,
11245                pid = child.pid,
11246                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11247                reason = ?final_state,
11248                ?stop_notice,
11249                "drain budget expired before the module exited; killing it"
11250            );
11251            child.start_kill().map_err(|source| {
11252                fail_after_wait_error(
11253                    module_id,
11254                    snapshot,
11255                    terminal_ring,
11256                    spawn_events,
11257                    format!("failed to kill during drain: {source}"),
11258                    failure,
11259                );
11260                SuperviseError::Kill {
11261                    module_id: module_id.to_string(),
11262                    source,
11263                }
11264            })?;
11265            let status = child.wait().await.map_err(|source| {
11266                fail_after_wait_error(
11267                    module_id,
11268                    snapshot,
11269                    terminal_ring,
11270                    spawn_events,
11271                    format!("failed to wait after drain kill: {source}"),
11272                    failure,
11273                );
11274                SuperviseError::Wait {
11275                    module_id: module_id.to_string(),
11276                    source,
11277                }
11278            })?;
11279            classify_reaped_child_exit(snapshot, &child, &status)
11280        }
11281    };
11282
11283    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11284    publish_after_terminal_record(
11285        module_id,
11286        || {
11287            record_terminal_with_detail(
11288                module_id,
11289                terminal_ring,
11290                spawn_events,
11291                &exit_report,
11292                terminal_disposition(final_state),
11293                detail,
11294            )
11295        },
11296        || {
11297            update_snapshot(snapshot, Some(module_id), |state| {
11298                state.state = final_state;
11299                if final_state == ModuleState::Failed {
11300                    state.failed_retry.failed_at = Some(Instant::now());
11301                }
11302                if let Some(enabled) = enabled {
11303                    state.enabled = enabled;
11304                }
11305                clear_current_process_facts(state);
11306                state.last_exit = Some(exit_report.clone());
11307                if exit_report.kind == ExitKind::DeliberateSeverance {
11308                    state.lifetime_restarts += 1;
11309                }
11310            })
11311        },
11312    )?;
11313    child.drain_stderr(module_id).await;
11314
11315    release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11316}
11317
11318/// Ask a child that nothing else has asked to stop, by signal.
11319///
11320/// A registered subc module is asked over its own connection: the drain sends
11321/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11322/// module GOODBYE, and the module stops itself. A module that speaks no subc
11323/// wire receives none of that, and neither does a subc module that has not
11324/// registered yet, so for them the drain budget would be pure delay in front of
11325/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11326/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11327/// into a recovery on the next start.
11328///
11329/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11330/// rule rather than an optimisation: that module's graceful stop is already
11331/// running by the time its child is drained, and a signal would race it.
11332///
11333/// Best-effort by construction. A child that has already exited is the ordinary
11334/// case rather than an error (the kill lands on a reaped or exiting pid), so a
11335/// failure is logged at debug and the wait-then-kill below still decides the
11336/// outcome.
11337#[cfg(unix)]
11338fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11339    let Some(pid) = child
11340        .id()
11341        .and_then(|pid| i32::try_from(pid).ok())
11342        .and_then(rustix::process::Pid::from_raw)
11343    else {
11344        debug!(
11345            module_id,
11346            "no pid to signal for teardown; falling through to the drain wait"
11347        );
11348        return;
11349    };
11350    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11351        Ok(()) => debug!(
11352            module_id,
11353            "sent SIGTERM to a module nothing else asked to stop"
11354        ),
11355        Err(err) => debug!(
11356            module_id,
11357            error = %err,
11358            "SIGTERM to module failed; the drain wait and kill still apply"
11359        ),
11360    }
11361}
11362
11363/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11364/// Windows does offer need cooperation this supervisor cannot assume: a console
11365/// control event requires sharing a console with the child, and `WM_CLOSE`
11366/// requires the child to pump a message loop. A supervised server process does
11367/// neither, so there is nothing to send and teardown is the wait followed by the
11368/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11369/// the thing `protocol: "none"` exists to avoid.
11370#[cfg(not(unix))]
11371fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11372    debug!(
11373        module_id,
11374        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11375    );
11376}
11377
11378fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11379    match final_state {
11380        ModuleState::Stopped => TerminalDisposition::Stopped,
11381        ModuleState::Disabled => TerminalDisposition::Disabled,
11382        ModuleState::Restarting => TerminalDisposition::Restarting,
11383        ModuleState::Failed => TerminalDisposition::Failed,
11384        ModuleState::Starting
11385        | ModuleState::Running
11386        | ModuleState::Unresponsive
11387        | ModuleState::Draining => {
11388            unreachable!("terminal exits only finish in terminal or restarting states")
11389        }
11390    }
11391}
11392
11393/// Release a reaped child's registration before allowing another spawn.
11394///
11395/// EOF is not a process-lifetime signal: an inherited socket can stay open
11396/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11397/// reading EOF. After the normal release grace, request connection close (which
11398/// cancels both reads and dispatch), then allow one more release grace for the
11399/// connection guard's forwarding cleanup. Never evict a different connection.
11400async fn release_dead_registration(
11401    registry: &Registry,
11402    forwarding: Option<&ForwardingTable>,
11403    snapshot: &SharedSnapshot,
11404    module_id: &str,
11405    failure: &FailureContext,
11406) -> Result<(), SuperviseError> {
11407    let result = async {
11408        let registration = registry
11409            .get_module(module_id)
11410            .map_err(SuperviseError::Registry)?;
11411        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11412            Ok(()) => return Ok(()),
11413            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11414            Err(err) => return Err(err),
11415        }
11416        let pid = lock_snapshot(snapshot)?.reaped_pid;
11417        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11418            warn!(
11419                module_id,
11420                pid,
11421                connection_id = registration.connection_id.get(),
11422                "reaped module registration outlived release grace; closing dead connection"
11423            );
11424            forwarding.request_connection_close(
11425                registration.connection_id,
11426                CloseReason::new(
11427                    "supervised_process_reaped",
11428                    format!("module '{module_id}' pid {pid} exited"),
11429                ),
11430            );
11431            wait_for_slot_registration_release(
11432                registry,
11433                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11434                REGISTRY_RELEASE_TIMEOUT,
11435            )
11436            .await?;
11437        }
11438        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11439    }
11440    .await;
11441    if let Err(err) = &result {
11442        fail_snapshot(snapshot, Some(module_id), None, failure);
11443        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11444    }
11445    result
11446}
11447
11448/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11449/// plain stop or restart waits for before it spawns a replacement.
11450async fn wait_for_registration_release(
11451    registry: &Registry,
11452    module_id: &str,
11453    wait: Duration,
11454) -> Result<(), SuperviseError> {
11455    wait_for_slot_registration_release(
11456        registry,
11457        crate::registry::RegistrationSlot::Active(module_id),
11458        wait,
11459    )
11460    .await
11461}
11462
11463/// Wait for the registration in `slot` to go away.
11464///
11465/// Keyed on the slot rather than the bare module id because a successful swap
11466/// never empties the id's active slot (the promoted candidate is in it), so an
11467/// id-keyed wait for the incumbent's release would always time out. Draining a
11468/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11469/// incumbent's connection instead.
11470async fn wait_for_slot_registration_release(
11471    registry: &Registry,
11472    slot: crate::registry::RegistrationSlot<'_>,
11473    wait: Duration,
11474) -> Result<(), SuperviseError> {
11475    let deadline = Instant::now() + wait;
11476    let mut release_events = registration_release_events().subscribe();
11477    let still_active = |registration: &crate::registry::ModuleRegistration| {
11478        SuperviseError::RegistrationStillActive {
11479            module_id: registration.manifest.module_id.clone(),
11480            waited: wait,
11481        }
11482    };
11483    loop {
11484        let _observed_generation = *release_events.borrow_and_update();
11485        let Some(registration) = registry
11486            .registration(slot)
11487            .map_err(SuperviseError::Registry)?
11488        else {
11489            return Ok(());
11490        };
11491
11492        let now = Instant::now();
11493        if now >= deadline {
11494            return Err(still_active(&registration));
11495        }
11496
11497        let remaining = deadline.saturating_duration_since(now);
11498        match timeout(remaining, release_events.changed()).await {
11499            Ok(Ok(())) | Ok(Err(_)) => {}
11500            Err(_) => return Err(still_active(&registration)),
11501        }
11502    }
11503}
11504
11505#[cfg(test)]
11506mod slot_registration_wait_tests {
11507    use super::*;
11508    use crate::registry::{ConnectionId, RegistrationSlot};
11509    use subc_protocol::manifest::ModuleManifest;
11510
11511    #[tokio::test]
11512    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11513        let registry = Arc::new(Registry::default());
11514        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11515        let runtime = supervisor.runtime_config();
11516        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11517        let spec = ModuleSpec {
11518            module_id: "enable-stale-registration".to_string(),
11519            program: PathBuf::from("/missing/enable-retry-test"),
11520            args: Vec::new(),
11521            env: Vec::new(),
11522            reserved: false,
11523            reserved_prefixes: Vec::new(),
11524            protocol: ModuleProtocol::Subc,
11525            overlap: Default::default(),
11526        };
11527        let connection = ConnectionId::new(90);
11528        registry
11529            .register_with_control_ops(
11530                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11531                1,
11532                connection,
11533                Vec::new(),
11534            )
11535            .unwrap();
11536        let mut child = None;
11537        let err = set_child_enabled(
11538            &spec,
11539            &runtime,
11540            &registry,
11541            &supervisor.process_liveness,
11542            &snapshot,
11543            &mut child,
11544            true,
11545            &runtime.failure_context(false),
11546        )
11547        .await
11548        .unwrap_err();
11549        assert!(matches!(
11550            err,
11551            SuperviseError::RegistrationStillActive { .. }
11552        ));
11553        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11554        assert!(child.is_none());
11555        registry.deregister_connection(connection).unwrap();
11556        let err = set_child_enabled(
11557            &spec,
11558            &runtime,
11559            &registry,
11560            &supervisor.process_liveness,
11561            &snapshot,
11562            &mut child,
11563            true,
11564            &runtime.failure_context(false),
11565        )
11566        .await
11567        .unwrap_err();
11568        assert!(
11569            matches!(err, SuperviseError::Spawn { .. }),
11570            "second enable must attempt a spawn: {err}"
11571        );
11572        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11573    }
11574
11575    const INCUMBENT: u64 = 1;
11576    const CANDIDATE: u64 = 2;
11577
11578    fn swapped_registry() -> Arc<Registry> {
11579        let registry = Arc::new(Registry::default());
11580        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11581        registry
11582            .register_with_control_ops(
11583                manifest.clone(),
11584                1,
11585                ConnectionId::new(INCUMBENT),
11586                Vec::new(),
11587            )
11588            .unwrap();
11589        registry
11590            .register_candidate_with_control_ops(
11591                manifest,
11592                1,
11593                ConnectionId::new(CANDIDATE),
11594                Vec::new(),
11595            )
11596            .unwrap();
11597        registry
11598    }
11599
11600    /// After a promotion the id's active slot is held by the new process, so an
11601    /// id-keyed wait for the incumbent's release can never succeed; the
11602    /// connection-keyed wait completes as soon as the incumbent deregisters.
11603    #[tokio::test]
11604    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11605        let registry = swapped_registry();
11606        registry.promote_candidate("m").unwrap().unwrap();
11607
11608        assert!(matches!(
11609            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11610            Err(SuperviseError::RegistrationStillActive { .. })
11611        ));
11612
11613        // Still held while the incumbent's connection has not deregistered.
11614        assert!(matches!(
11615            wait_for_slot_registration_release(
11616                &registry,
11617                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11618                Duration::from_millis(50),
11619            )
11620            .await,
11621            Err(SuperviseError::RegistrationStillActive { .. })
11622        ));
11623
11624        let releaser = Arc::clone(&registry);
11625        let release = tokio::spawn(async move {
11626            sleep(Duration::from_millis(20)).await;
11627            releaser
11628                .deregister_connection(ConnectionId::new(INCUMBENT))
11629                .unwrap();
11630            notify_registration_release();
11631        });
11632        wait_for_slot_registration_release(
11633            &registry,
11634            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11635            Duration::from_secs(5),
11636        )
11637        .await
11638        .expect("the incumbent's own registration is released");
11639        release.await.unwrap();
11640        assert!(registry.get_module("m").unwrap().is_some());
11641    }
11642
11643    /// The candidate slot is waited on separately from the active slot: the
11644    /// incumbent's registration neither holds up nor stands in for it.
11645    #[tokio::test]
11646    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11647        let registry = swapped_registry();
11648        assert!(matches!(
11649            wait_for_slot_registration_release(
11650                &registry,
11651                RegistrationSlot::Candidate("m"),
11652                Duration::from_millis(50),
11653            )
11654            .await,
11655            Err(SuperviseError::RegistrationStillActive { .. })
11656        ));
11657        registry
11658            .deregister_connection(ConnectionId::new(CANDIDATE))
11659            .unwrap();
11660        wait_for_slot_registration_release(
11661            &registry,
11662            RegistrationSlot::Candidate("m"),
11663            Duration::from_millis(50),
11664        )
11665        .await
11666        .expect("a candidate slot with no candidate is released");
11667        assert!(registry
11668            .registration(RegistrationSlot::Active("m"))
11669            .unwrap()
11670            .is_some());
11671    }
11672}
11673
11674fn classify_exit(status: &ExitStatus) -> ExitReport {
11675    ExitReport {
11676        kind: if status.success() {
11677            ExitKind::Clean
11678        } else {
11679            ExitKind::Crash
11680        },
11681        code: status.code(),
11682        signal: exit_signal(status),
11683        at_ms: unix_ms_now(),
11684    }
11685}
11686
11687/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11688/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11689/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11690/// disposition still must be `Failed` so the terminal ring is not silently missing
11691/// an entry, matching what `fail_snapshot` records for this same arm.
11692fn wait_error_exit_report() -> ExitReport {
11693    ExitReport {
11694        kind: ExitKind::Crash,
11695        code: None,
11696        signal: None,
11697        at_ms: unix_ms_now(),
11698    }
11699}
11700
11701#[cfg(unix)]
11702fn exit_signal(status: &ExitStatus) -> Option<i32> {
11703    use std::os::unix::process::ExitStatusExt;
11704
11705    status.signal()
11706}
11707
11708#[cfg(not(unix))]
11709fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11710    None
11711}
11712
11713/// Give an operator-touched module its full crash budget back.
11714///
11715/// Named for the counter it used to zero; it now empties the in-window ring,
11716/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11717/// ledger of what happened survives every operator action.
11718fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11719    update_snapshot(snapshot, Some(module_id), |state| {
11720        state.clear_crash_restarts();
11721    })
11722}
11723
11724fn set_running(
11725    snapshot: &SharedSnapshot,
11726    child: &SupervisedChild,
11727    module_id: &str,
11728    spawn_events: &SpawnEventFeed,
11729) -> Result<(), SuperviseError> {
11730    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11731        module_id: Some(module_id.to_string()),
11732    })?;
11733    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11734    if std::mem::take(&mut state.coalesced_restart_pending) {
11735        let generation = state.spawn_generation;
11736        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11737    }
11738    state.drain_disposition_detail = None;
11739    state.spawn_failure = None;
11740    state.waiting_for_previous_pid = None;
11741    state.waiting_for_previous_detail = None;
11742    // Every caller of this is a plain spawn, which always uses the primary key;
11743    // a promoted swap candidate sets the flag itself after this returns.
11744    state.in_alternate_slot = false;
11745    state.configuration_updated_since_spawn = false;
11746    state.spawned_protocol = Some(child.protocol);
11747    state.state = ModuleState::Running;
11748    state.enabled = true;
11749    state.process_alive = true;
11750    state.pid = child.id();
11751    // Probe samples belong to one process. Keep module-level escalation and
11752    // late-answer history, but never charge a new child for its predecessor's misses.
11753    state.health.status = SupervisorHealthStatus::Unknown;
11754    state.health.consecutive_failures = 0;
11755    state.health.detail = None;
11756    state.health.metrics = None;
11757    state.health.last_probe_ms = None;
11758    state.failed_retry.healthy_since = None;
11759    state.failed_retry.failed_at = None;
11760    state.failed_retry.running_since = Some(child.spawned_at);
11761    #[cfg(windows)]
11762    {
11763        state.pipe_consumed = child
11764            .nonce_delivery
11765            .as_ref()
11766            .map(|delivery| delivery.consumed());
11767    }
11768    #[cfg(target_os = "macos")]
11769    {
11770        state.report_ready = Some(Arc::clone(&child.report_ready));
11771    }
11772    state.spawned_at_ms = Some(child.spawned_at_ms);
11773    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11774    state.spawned_from = Some(child.spawned_from.clone());
11775    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11776    state.spawned_file_identity = child.spawned_file_identity.clone();
11777    state.process_start_time = child.process_start_time;
11778    Ok(())
11779}
11780
11781fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11782    state.failed_retry.healthy_since = None;
11783    state.failed_retry.running_since = None;
11784    state.waiting_for_previous_pid = None;
11785    state.waiting_for_previous_detail = None;
11786    state.process_alive = false;
11787    state.spawned_protocol = None;
11788    state.pid = None;
11789    #[cfg(windows)]
11790    {
11791        state.pipe_consumed = None;
11792    }
11793    #[cfg(target_os = "macos")]
11794    {
11795        state.report_ready = None;
11796    }
11797    state.spawned_at_ms = None;
11798    state.exec_outcome = None;
11799    state.spawned_from = None;
11800    state.launch_spec_digest = None;
11801    state.spawned_file_identity = None;
11802    state.process_start_time = None;
11803    state.deliberate_severance = None;
11804}
11805
11806#[cfg(test)]
11807fn record_deliberate_severance(
11808    snapshot: &SharedSnapshot,
11809    identity: ProcessIdentity,
11810) -> Result<(), SuperviseError> {
11811    update_snapshot(snapshot, None, |state| {
11812        state.deliberate_severance = Some(identity);
11813    })
11814}
11815
11816fn apply_deliberate_severance_marker(
11817    snapshot: &SharedSnapshot,
11818    exited_identity: Option<ProcessIdentity>,
11819    mut exit_report: ExitReport,
11820) -> ExitReport {
11821    let marker = lock_snapshot(snapshot)
11822        .ok()
11823        .and_then(|mut state| state.deliberate_severance.take());
11824    if marker.is_some() && marker == exited_identity {
11825        exit_report.kind = ExitKind::DeliberateSeverance;
11826    }
11827    exit_report
11828}
11829
11830fn classify_reaped_child_exit(
11831    snapshot: &SharedSnapshot,
11832    child: &SupervisedChild,
11833    status: &ExitStatus,
11834) -> ExitReport {
11835    let _ = update_snapshot(snapshot, None, |state| {
11836        state.reaped_pid = Some(child.pid);
11837        state.spawn_failure = child.spawn_failure.clone();
11838    });
11839    if child.spawn_failure.as_deref() == Some("registration_timeout") {
11840        registration_failure_exit_report(classify_exit(status))
11841    } else {
11842        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11843    }
11844}
11845
11846fn fail_snapshot(
11847    snapshot: &SharedSnapshot,
11848    module_id: Option<&str>,
11849    last_exit: Option<ExitReport>,
11850    failure: &FailureContext,
11851) {
11852    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11853        state.state = ModuleState::Failed;
11854        clear_current_process_facts(state);
11855        if failure.may_retry() && state.enabled {
11856            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11857        } else if failure.shutdown.is_set() {
11858            state.failed_retry.cancel();
11859        }
11860        if let Some(last_exit) = last_exit {
11861            state.last_exit = Some(last_exit);
11862        }
11863    }) {
11864        error!(error = %err, "failed to mark supervisor state failed");
11865    }
11866}
11867
11868fn update_snapshot(
11869    snapshot: &SharedSnapshot,
11870    module_id: Option<&str>,
11871    update: impl FnOnce(&mut SupervisorSnapshot),
11872) -> Result<(), SuperviseError> {
11873    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11874        module_id: module_id.map(ToOwned::to_owned),
11875    })?;
11876    update(&mut state);
11877    Ok(())
11878}
11879
11880const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11881
11882fn lock_snapshot_for_control<'a>(
11883    snapshot: &'a SharedSnapshot,
11884    module_id: &str,
11885    caller: &'static str,
11886) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11887    let started_at = Instant::now();
11888    let guard = lock_snapshot(snapshot)?;
11889    let waited = started_at.elapsed();
11890    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11891        warn!(
11892            module_id = %module_id,
11893            waited_ms = waited.as_millis() as u64,
11894            caller = %caller,
11895            "slow snapshot lock"
11896        );
11897    }
11898    Ok(guard)
11899}
11900
11901fn lock_snapshot(
11902    snapshot: &SharedSnapshot,
11903) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11904    snapshot
11905        .lock()
11906        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11907}
11908
11909#[cfg(test)]
11910mod terminal_history_tests {
11911    use std::{
11912        path::PathBuf,
11913        sync::Arc,
11914        time::{Duration, Instant},
11915    };
11916
11917    use tokio::{sync::mpsc, time::sleep};
11918
11919    use super::{
11920        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11921        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11922        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11923        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11924        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11925        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11926        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11927        SupervisorHealthStatus, SupervisorSnapshot,
11928    };
11929    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11930    // use for their own wall-clock deadlines: crash-restart instants must be on
11931    // the same clock the production code stamps them with, which is tokio's (and
11932    // is what `start_paused` tests can move).
11933    use super::Instant as ClockInstant;
11934    use crate::{
11935        registry::Registry,
11936        terminal_ring::{TerminalRing, TerminalRingConfig},
11937    };
11938    use std::sync::Mutex;
11939    use subc_control::TerminalDisposition;
11940
11941    /// See the twin in `control.rs` for why this derives the path from
11942    /// `current_exe()` and why the existence check is here: `--lib` alone does
11943    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11944    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11945    pub(super) fn fake_aft_stub_path() -> PathBuf {
11946        let mut path = std::env::current_exe().expect("current_exe available in tests");
11947        path.pop();
11948        path.pop();
11949        path.push(if cfg!(windows) {
11950            "fake-aft-stub.exe"
11951        } else {
11952            "fake-aft-stub"
11953        });
11954        assert!(
11955            path.exists(),
11956            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11957             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11958            path.display()
11959        );
11960        path
11961    }
11962
11963    #[test]
11964    fn reserved_never_spawned_refuses_every_hello() {
11965        // The canary hole: a reserved id whose module has never spawned had NO
11966        // gate entry and admitted anyone -- the reservation protected the nonce
11967        // holder, not the NAME. Now the entry is present with no legitimate
11968        // holder and refuses all comers.
11969        let supervisor = SupervisorHandle::default();
11970        supervisor.apply_identity_configuration(&ModuleSpec {
11971            module_id: "never-spawned".to_string(),
11972            program: PathBuf::from("/usr/bin/false"),
11973            args: Vec::new(),
11974            env: Vec::new(),
11975            reserved: true,
11976            reserved_prefixes: Vec::new(),
11977            protocol: ModuleProtocol::Subc,
11978            overlap: Default::default(),
11979        });
11980        assert!(
11981            supervisor
11982                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11983                .is_err(),
11984            "forged nonce must refuse on a reserved never-spawned id"
11985        );
11986        assert!(
11987            supervisor
11988                .reserved_hello_verification("never-spawned", None)
11989                .is_err(),
11990            "absent nonce must refuse on a reserved never-spawned id"
11991        );
11992        // And a real spawn nonce minted later admits exactly that nonce.
11993        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11994        supervisor.apply_identity_configuration(&ModuleSpec {
11995            module_id: "never-spawned".to_string(),
11996            program: PathBuf::from("/usr/bin/false"),
11997            args: Vec::new(),
11998            env: Vec::new(),
11999            reserved: true,
12000            reserved_prefixes: Vec::new(),
12001            protocol: ModuleProtocol::Subc,
12002            overlap: Default::default(),
12003        });
12004        assert!(supervisor
12005            .reserved_hello_verification("never-spawned", Some("minted"))
12006            .is_ok());
12007        assert!(supervisor
12008            .reserved_hello_verification("never-spawned", Some("forged"))
12009            .is_err());
12010    }
12011
12012    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12013    /// happened, which is what "spent budget" looks like to every reader.
12014    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12015        let now = ClockInstant::now();
12016        for _ in 0..count {
12017            state.crash_restarts.push_back(now);
12018        }
12019    }
12020
12021    /// Age the oldest recorded restart out of `window`, standing in for the hours
12022    /// that would otherwise have to pass. Injecting the instant is the point: a
12023    /// test that slept a real window would take ten minutes and still prove less.
12024    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12025        let aged = state
12026            .crash_restarts
12027            .front()
12028            .expect("a crash restart must be recorded before it can be aged")
12029            .checked_sub(window + Duration::from_secs(1))
12030            .expect("the test clock is far enough from its origin to age an instant");
12031        state.crash_restarts[0] = aged;
12032    }
12033
12034    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12035        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12036        seed_crash_restarts(&mut state, count);
12037        state
12038    }
12039
12040    #[test]
12041    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12042        let policy = RestartPolicy::new(3, Duration::ZERO);
12043        let now = ClockInstant::now();
12044        assert!(daemon_will_restart(
12045            &mut snapshot_with_restarts(true, 2),
12046            &policy,
12047            now
12048        ));
12049        assert!(!daemon_will_restart(
12050            &mut snapshot_with_restarts(true, 3),
12051            &policy,
12052            now
12053        ));
12054        assert!(!daemon_will_restart(
12055            &mut snapshot_with_restarts(false, 0),
12056            &policy,
12057            now
12058        ));
12059    }
12060
12061    #[test]
12062    fn crash_restart_backoff_escalates_with_in_window_count() {
12063        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12064            .with_max_backoff(Duration::from_secs(30));
12065        let now = ClockInstant::now();
12066        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12067        let schedules = (0..4)
12068            .map(|_| {
12069                state
12070                    .next_crash_restart(&policy, now)
12071                    .expect("the test policy allows four crash restarts")
12072            })
12073            .collect::<Vec<_>>();
12074
12075        assert_eq!(
12076            schedules
12077                .iter()
12078                .map(|schedule| schedule.restart_in_window)
12079                .collect::<Vec<_>>(),
12080            vec![0, 1, 2, 3]
12081        );
12082        assert_eq!(
12083            schedules
12084                .iter()
12085                .map(|schedule| schedule.delay)
12086                .collect::<Vec<_>>(),
12087            vec![
12088                Duration::from_millis(100),
12089                Duration::from_secs(1),
12090                Duration::from_secs(10),
12091                Duration::from_secs(30),
12092            ]
12093        );
12094    }
12095
12096    #[test]
12097    fn crash_restart_backoff_resets_after_ring_clear() {
12098        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12099        let now = ClockInstant::now();
12100        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12101        assert_eq!(
12102            state.next_crash_restart(&policy, now).unwrap().delay,
12103            Duration::from_millis(100)
12104        );
12105        assert_eq!(
12106            state.next_crash_restart(&policy, now).unwrap().delay,
12107            Duration::from_secs(1)
12108        );
12109
12110        state.clear_crash_restarts();
12111        let schedule = state
12112            .next_crash_restart(&policy, now)
12113            .expect("a cleared ring must allow another restart");
12114        assert_eq!(schedule.restart_in_window, 0);
12115        assert_eq!(schedule.delay, Duration::from_millis(100));
12116    }
12117
12118    #[test]
12119    fn crash_restart_backoff_ignores_aged_restarts() {
12120        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12121        let now = ClockInstant::now();
12122        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12123        state
12124            .next_crash_restart(&policy, now)
12125            .expect("the first restart is allowed");
12126        state
12127            .next_crash_restart(&policy, now)
12128            .expect("the second restart is allowed");
12129        state.crash_restarts[0] = now
12130            .checked_sub(policy.window + Duration::from_secs(1))
12131            .expect("the fake clock can age a restart past the window");
12132
12133        let schedule = state
12134            .next_crash_restart(&policy, now)
12135            .expect("an aged restart must release its slot");
12136        assert_eq!(schedule.restart_in_window, 1);
12137        assert_eq!(schedule.delay, Duration::from_secs(1));
12138        assert_eq!(state.crash_restarts.len(), 2);
12139    }
12140
12141    /// The budget is a rate: the same three spent restarts refuse a respawn
12142    /// while they are recent and allow one once they have aged past the window.
12143    /// Nothing about the module changed in between, which is the whole point.
12144    #[test]
12145    fn a_budget_spent_before_the_window_no_longer_refuses() {
12146        let policy = RestartPolicy::new(3, Duration::ZERO);
12147        let mut state = snapshot_with_restarts(true, 3);
12148        let now = ClockInstant::now();
12149        assert!(!daemon_will_restart(&mut state, &policy, now));
12150
12151        assert!(daemon_will_restart(
12152            &mut state,
12153            &policy,
12154            now + policy.window + Duration::from_secs(1)
12155        ));
12156        assert!(
12157            state.crash_restarts.is_empty(),
12158            "reading the budget must drop the instants that left the window"
12159        );
12160    }
12161
12162    fn module_with_recovery_snapshot(
12163        state: ModuleState,
12164        enabled: bool,
12165        restart_count: u32,
12166    ) -> SupervisedModule {
12167        let registry = Arc::new(Registry::default());
12168        let supervisor =
12169            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12170        let runtime = supervisor.runtime_config();
12171        let spec = ModuleSpec {
12172            module_id: "recovery-snapshot".to_string(),
12173            program: fake_aft_stub_path(),
12174            args: Vec::new(),
12175            env: Vec::new(),
12176            reserved: false,
12177            reserved_prefixes: Vec::new(),
12178            protocol: ModuleProtocol::Subc,
12179            overlap: Default::default(),
12180        };
12181        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12182        seed_crash_restarts(&mut snapshot, restart_count);
12183        // These tests read synthetic snapshots. A real child and monitor would
12184        // race those reads by replacing the requested state during startup.
12185        let (commands, _rx) = mpsc::channel(4);
12186        SupervisedModule {
12187            inner: Arc::new(SupervisedModuleInner {
12188                module_id: spec.module_id.clone(),
12189                registry,
12190                snapshot: Arc::new(Mutex::new(snapshot)),
12191                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12192                    spec,
12193                    health: runtime.health,
12194                })),
12195                stderr_ring: runtime.stderr_ring,
12196                terminal_ring: runtime.terminal_ring,
12197                commands,
12198                monitor: Mutex::new(None),
12199                restart_policy: runtime.restart_policy,
12200                effective_drain_timeout: runtime.effective_drain_timeout,
12201                provenance_probe: supervisor.provenance_probe.clone(),
12202            }),
12203        }
12204    }
12205
12206    #[cfg(target_os = "linux")]
12207    #[tokio::test]
12208    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12209        let supervisor =
12210            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12211                .with_cgroup_placement(None);
12212        let result = supervisor.spawn(ModuleSpec {
12213            module_id: "no-cgroup-placement".to_string(),
12214            program: fake_aft_stub_path(),
12215            args: Vec::new(),
12216            env: Vec::new(),
12217            reserved: false,
12218            reserved_prefixes: Vec::new(),
12219            protocol: ModuleProtocol::Subc,
12220            overlap: Default::default(),
12221        });
12222
12223        assert!(
12224            result.is_ok(),
12225            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12226        );
12227    }
12228
12229    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12230    async fn undecided_snapshot_uses_shared_restart_predicate() {
12231        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12232            .will_recover_after_connection_loss()
12233            .unwrap());
12234        assert!(
12235            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12236                .will_recover_after_connection_loss()
12237                .unwrap()
12238        );
12239    }
12240
12241    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12242    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12243        assert!(
12244            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12245                .will_recover_after_connection_loss()
12246                .unwrap()
12247        );
12248    }
12249
12250    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12251    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12252        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12253            .will_recover_after_connection_loss()
12254            .unwrap());
12255        assert!(
12256            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12257                .will_recover_after_connection_loss()
12258                .unwrap()
12259        );
12260    }
12261
12262    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12263    async fn warming_snapshot_is_limited_to_startup_phases() {
12264        for state in [
12265            ModuleState::Starting,
12266            ModuleState::Running,
12267            ModuleState::Restarting,
12268        ] {
12269            assert!(
12270                module_with_recovery_snapshot(state, true, 0)
12271                    .is_warming()
12272                    .unwrap(),
12273                "{state:?} should be warming"
12274            );
12275        }
12276        for state in [
12277            ModuleState::Unresponsive,
12278            ModuleState::Draining,
12279            ModuleState::Stopped,
12280            ModuleState::Failed,
12281            ModuleState::Disabled,
12282        ] {
12283            assert!(
12284                !module_with_recovery_snapshot(state, true, 0)
12285                    .is_warming()
12286                    .unwrap(),
12287                "{state:?} should not be warming"
12288            );
12289        }
12290    }
12291
12292    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12293    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12294        let registry = Arc::new(Registry::default());
12295        let supervisor =
12296            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12297        let module = supervisor
12298            .spawn(ModuleSpec {
12299                module_id: "terminal-history".to_string(),
12300                program: fake_aft_stub_path(),
12301                args: Vec::new(),
12302                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12303                reserved: false,
12304                reserved_prefixes: Vec::new(),
12305                protocol: ModuleProtocol::Subc,
12306                overlap: Default::default(),
12307            })
12308            .unwrap();
12309
12310        let deadline = Instant::now() + Duration::from_secs(5);
12311        loop {
12312            let history = module.terminal_history();
12313            if history.entries.len() == 2 {
12314                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12315                assert_eq!(history.dropped, 0);
12316                assert_eq!(
12317                    history
12318                        .entries
12319                        .iter()
12320                        .map(|entry| entry.exit_code)
12321                        .collect::<Vec<_>>(),
12322                    vec![Some(23), Some(23)]
12323                );
12324                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12325                return;
12326            }
12327            assert!(
12328                Instant::now() < deadline,
12329                "module did not retain two terminal exits: {history:?}"
12330            );
12331            sleep(Duration::from_millis(10)).await;
12332        }
12333    }
12334
12335    /// A disable issued while a crash respawn is still backing off must preempt
12336    /// that respawn: the operator's stop wins, the disable must not queue behind
12337    /// the backoff, and the module must never come back up afterwards.
12338    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12339    async fn disable_during_crash_backoff_cancels_pending_respawn() {
12340        let backoff = Duration::from_secs(2);
12341        let supervisor = Supervisor::new_for_test(
12342            Arc::new(Registry::default()),
12343            RestartPolicy::new(10, backoff),
12344        );
12345        let module = supervisor
12346            .spawn(ModuleSpec {
12347                module_id: "disable-during-backoff".to_string(),
12348                program: fake_aft_stub_path(),
12349                args: Vec::new(),
12350                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12351                reserved: false,
12352                reserved_prefixes: Vec::new(),
12353                protocol: ModuleProtocol::Subc,
12354                overlap: Default::default(),
12355            })
12356            .unwrap();
12357
12358        // Wait for the first crash to put the module into its backoff window.
12359        let deadline = Instant::now() + Duration::from_secs(5);
12360        loop {
12361            if module.status().unwrap().state == ModuleState::Restarting {
12362                break;
12363            }
12364            assert!(
12365                Instant::now() < deadline,
12366                "module never entered the crash backoff"
12367            );
12368            sleep(Duration::from_millis(10)).await;
12369        }
12370
12371        let started = Instant::now();
12372        module.set_enabled(false).await.unwrap();
12373        let waited = started.elapsed();
12374
12375        assert!(
12376            waited < backoff / 2,
12377            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12378        );
12379        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12380
12381        // Outlast the backoff: the respawn it was counting down to must never run.
12382        sleep(backoff + Duration::from_millis(500)).await;
12383        let status = module.status().unwrap();
12384        assert_eq!(status.state, ModuleState::Disabled);
12385        assert_eq!(
12386            status.spawn_generation, 1,
12387            "module respawned after the operator disabled it"
12388        );
12389    }
12390
12391    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12392    /// the shape of nats-server, the program this rule exists for.
12393    #[cfg(unix)]
12394    fn protocol_none_sigterm_exits_clean_spec(
12395        module_id: &str,
12396        dir: &std::path::Path,
12397    ) -> (ModuleSpec, PathBuf, PathBuf) {
12398        let ready = dir.join("ready");
12399        let marker = dir.join("sigterm");
12400        let spec = ModuleSpec {
12401            module_id: module_id.to_string(),
12402            program: fake_aft_stub_path(),
12403            args: Vec::new(),
12404            env: vec![
12405                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12406                (
12407                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12408                    marker.display().to_string(),
12409                ),
12410                (
12411                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12412                    ready.display().to_string(),
12413                ),
12414            ],
12415            reserved: false,
12416            reserved_prefixes: Vec::new(),
12417            protocol: ModuleProtocol::None,
12418            overlap: Default::default(),
12419        };
12420        (spec, ready, marker)
12421    }
12422
12423    /// Wait for a file the child writes, so a signal is never sent before the
12424    /// child's SIGTERM handler is installed (the default disposition would
12425    /// kill it by signal and the exit would not be clean).
12426    #[cfg(unix)]
12427    async fn wait_for_file(path: &std::path::Path) {
12428        let deadline = Instant::now() + Duration::from_secs(10);
12429        while !path.exists() {
12430            assert!(
12431                Instant::now() < deadline,
12432                "{} never appeared",
12433                path.display()
12434            );
12435            sleep(Duration::from_millis(10)).await;
12436        }
12437    }
12438
12439    /// A protocol-none module that exits 0 because something OUTSIDE the
12440    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12441    /// the crash-path disposition rather than `stopped`.
12442    #[cfg(unix)]
12443    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12444    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12445        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12446        let (spec, ready, marker) =
12447            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12448        let supervisor = Supervisor::new_for_test(
12449            Arc::new(Registry::default()),
12450            RestartPolicy::new(3, Duration::ZERO),
12451        );
12452        let module = supervisor.spawn(spec).unwrap();
12453        wait_for_file(&ready).await;
12454        // The ready file proves the child installed its SIGTERM handler, not
12455        // that the supervisor has processed the privacy trampoline's exec
12456        // acknowledgement. On macOS status withholds the pid until then.
12457        let deadline = Instant::now() + Duration::from_secs(10);
12458        let first_pid = loop {
12459            let status = module.status().unwrap();
12460            if status.state == ModuleState::Running {
12461                if let Some(pid) = status.pid {
12462                    break pid;
12463                }
12464            }
12465            assert!(
12466                Instant::now() < deadline,
12467                "a running module must report its pid after exec confirmation: {status:?}"
12468            );
12469            sleep(Duration::from_millis(10)).await;
12470        };
12471
12472        rustix::process::kill_process(
12473            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12474            rustix::process::Signal::TERM,
12475        )
12476        .unwrap();
12477
12478        let deadline = Instant::now() + Duration::from_secs(10);
12479        let respawned = loop {
12480            let status = module.status().unwrap();
12481            if status.state == ModuleState::Running
12482                && status.pid.is_some_and(|pid| pid != first_pid)
12483            {
12484                break status;
12485            }
12486            assert!(
12487                Instant::now() < deadline,
12488                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12489            );
12490            sleep(Duration::from_millis(10)).await;
12491        };
12492        assert_eq!(respawned.spawn_generation, 2);
12493        assert!(
12494            marker.exists(),
12495            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12496        );
12497
12498        let history = module.terminal_history();
12499        assert_eq!(history.entries.len(), 1, "{history:?}");
12500        let entry = &history.entries[0];
12501        assert_eq!(entry.exit_code, Some(0));
12502        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12503        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12504
12505        module.stop().await.unwrap();
12506    }
12507
12508    /// Repeated unrequested clean exits of a protocol-none module spend the
12509    /// restart budget exactly as crashes do, and the module ends `failed` with
12510    /// the budget named.
12511    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12512    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12513        let supervisor = Supervisor::new_for_test(
12514            Arc::new(Registry::default()),
12515            RestartPolicy::new(1, Duration::ZERO),
12516        );
12517        let module = supervisor
12518            .spawn(ModuleSpec {
12519                module_id: "none-clean-exit-budget".to_string(),
12520                program: fake_aft_stub_path(),
12521                args: Vec::new(),
12522                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12523                reserved: false,
12524                reserved_prefixes: Vec::new(),
12525                protocol: ModuleProtocol::None,
12526                overlap: Default::default(),
12527            })
12528            .unwrap();
12529
12530        // Failed follows the terminal write; this deadline only bounds a hang,
12531        // not an assumed duration for the two launches or their exit recording.
12532        let deadline = Instant::now() + Duration::from_secs(10);
12533        loop {
12534            let status = module.status().unwrap();
12535            if status.state == ModuleState::Failed {
12536                break;
12537            }
12538            assert!(
12539                Instant::now() < deadline,
12540                "module never exhausted its budget: {status:?} {:?}",
12541                module.terminal_history()
12542            );
12543            sleep(Duration::from_millis(10)).await;
12544        }
12545        let history = module.terminal_history();
12546        assert_eq!(
12547            history
12548                .entries
12549                .iter()
12550                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12551                .collect::<Vec<_>>(),
12552            vec![
12553                (Some(0), TerminalDisposition::Restarting),
12554                (Some(0), TerminalDisposition::Failed),
12555            ]
12556        );
12557        let detail = history.entries[1]
12558            .disposition_detail
12559            .as_deref()
12560            .expect("a budget failure names the budget");
12561        assert!(detail.contains("max_restarts=1"), "{detail}");
12562        assert_eq!(module.status().unwrap().spawn_generation, 2);
12563    }
12564
12565    #[test]
12566    fn restart_budget_failure_is_published_after_its_terminal_record() {
12567        use std::io::{Read, Write};
12568        let supervisor = Supervisor::new_for_test(
12569            Arc::new(Registry::default()),
12570            RestartPolicy::new(0, Duration::ZERO),
12571        );
12572        let runtime = supervisor.runtime_config();
12573        let spec = ModuleSpec {
12574            protocol: ModuleProtocol::None,
12575            ..windowed_crash_spec("budget-publication-order")
12576        };
12577        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12578            ModuleState::Running,
12579            true,
12580        )));
12581        lock_snapshot(&snapshot).unwrap().process_alive = true;
12582        let reaped_snapshot = snapshot.clone();
12583        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12584        let history = runtime.terminal_ring.clone();
12585        let reap = std::thread::spawn(move || {
12586            tokio::runtime::Builder::new_current_thread()
12587                .enable_all()
12588                .build()
12589                .unwrap()
12590                .block_on(on_child_exit(
12591                    &spec,
12592                    runtime.restart_policy,
12593                    &supervisor.registry,
12594                    &reaped_snapshot,
12595                    &runtime.terminal_ring,
12596                    &runtime.spawn_events,
12597                    &runtime.child_roster,
12598                    ExitReport {
12599                        kind: ExitKind::Clean,
12600                        code: Some(0),
12601                        signal: None,
12602                        at_ms: 1,
12603                    },
12604                    true,
12605                ))
12606        });
12607        let mut reached = [0];
12608        barrier.read_exact(&mut reached).unwrap();
12609        assert_eq!(&reached, b"R");
12610        let (before_state, before_alive) = {
12611            let state = lock_snapshot(&snapshot).unwrap();
12612            (state.state, state.process_alive)
12613        };
12614        let before_history = history.lock().unwrap().snapshot();
12615        barrier.write_all(b"X").unwrap();
12616        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12617        assert!(before_history.entries.is_empty());
12618        assert_ne!(
12619            before_state,
12620            ModuleState::Failed,
12621            "Failed was visible before its terminal record could be written"
12622        );
12623        assert!(before_alive);
12624        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12625        let history = history.lock().unwrap().snapshot();
12626        assert_eq!(history.entries.len(), 1);
12627        assert_eq!(history.entries[0].exit_code, Some(0));
12628        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12629    }
12630
12631    #[test]
12632    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12633        use std::io::{Read, Write};
12634
12635        let supervisor = Supervisor::new_for_test(
12636            Arc::new(Registry::default()),
12637            RestartPolicy::new(3, Duration::ZERO),
12638        );
12639        let mut runtime = supervisor.runtime_config();
12640        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12641        let snapshot = module.inner.snapshot.clone();
12642        lock_snapshot(&snapshot).unwrap().process_alive = true;
12643        runtime.terminal_ring = module.inner.terminal_ring.clone();
12644        let spec = windowed_crash_spec(module.module_id());
12645        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12646        let reaped_snapshot = snapshot.clone();
12647        let reap = std::thread::spawn(move || {
12648            tokio::runtime::Builder::new_current_thread()
12649                .enable_all()
12650                .build()
12651                .unwrap()
12652                .block_on(on_child_exit(
12653                    &spec,
12654                    runtime.restart_policy,
12655                    &supervisor.registry,
12656                    &reaped_snapshot,
12657                    &runtime.terminal_ring,
12658                    &runtime.spawn_events,
12659                    &runtime.child_roster,
12660                    ExitReport {
12661                        kind: ExitKind::Crash,
12662                        code: Some(2),
12663                        signal: None,
12664                        at_ms: 1,
12665                    },
12666                    true,
12667                ))
12668        });
12669
12670        // Hold the crash handler between its budget decision and publication.
12671        // Connection cleanup must still see the recovery that this crash chose,
12672        // even when that recovery consumes the last available restart slot.
12673        let mut reached = [0];
12674        barrier.read_exact(&mut reached).unwrap();
12675        assert_eq!(&reached, b"R");
12676        let before_status = module.status().unwrap();
12677        let before_history = module.terminal_history();
12678        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12679        barrier.write_all(b"X").unwrap();
12680        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12681
12682        assert_eq!(before_status.state, ModuleState::Running);
12683        assert!(before_status.process_alive);
12684        assert_eq!(before_status.restart_count, 3);
12685        assert!(before_history.entries.is_empty());
12686        assert!(
12687            before_recovers,
12688            "connection cleanup would report a terminal close for a scheduled crash restart"
12689        );
12690        let restarting = module.status().unwrap();
12691        assert_eq!(restarting.state, ModuleState::Restarting);
12692        assert_eq!(restarting.restart_count, 3);
12693        assert!(module.will_recover_after_connection_loss().unwrap());
12694        let history = module.terminal_history();
12695        assert_eq!(history.entries.len(), 1);
12696        assert_eq!(
12697            history.entries[0].disposition,
12698            TerminalDisposition::Restarting
12699        );
12700
12701        // Once the replacement is running, its next crash must be judged
12702        // against the exhausted budget rather than the previous recovery.
12703        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12704            state.state = ModuleState::Running;
12705            state.respawn_pending = false;
12706        })
12707        .unwrap();
12708        assert!(!module.will_recover_after_connection_loss().unwrap());
12709        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12710            state.enabled = false;
12711            state.respawn_pending = true;
12712        })
12713        .unwrap();
12714        assert!(!module.will_recover_after_connection_loss().unwrap());
12715    }
12716
12717    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12718        use std::net::{TcpListener, TcpStream};
12719        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12720        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12721        let (barrier, _) = listener.accept().unwrap();
12722        for stream in [&writer, &barrier] {
12723            stream
12724                .set_read_timeout(Some(Duration::from_secs(10)))
12725                .unwrap();
12726            stream
12727                .set_write_timeout(Some(Duration::from_secs(10)))
12728                .unwrap();
12729        }
12730        *events.1.lock().unwrap() = Some(writer);
12731        barrier
12732    }
12733
12734    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12735        use std::io::{Read, Write};
12736        let barrier = events.1.lock().unwrap().take();
12737        if let Some(mut barrier) = barrier {
12738            barrier.write_all(b"R").unwrap();
12739            let mut release = [0];
12740            barrier.read_exact(&mut release).unwrap();
12741            assert_eq!(&release, b"X");
12742        }
12743    }
12744
12745    #[test]
12746    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12747        use std::io::{Read, Write};
12748
12749        let supervisor = Supervisor::new_for_test(
12750            Arc::new(Registry::default()),
12751            RestartPolicy::new(0, Duration::ZERO),
12752        );
12753        let mut runtime = supervisor.runtime_config();
12754        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12755        let snapshot = module.inner.snapshot.clone();
12756        runtime.terminal_ring = module.inner.terminal_ring.clone();
12757        let spec = ModuleSpec {
12758            module_id: module.module_id().into(),
12759            program: fake_aft_stub_path(),
12760            protocol: ModuleProtocol::None,
12761            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12762            ..windowed_crash_spec(module.module_id())
12763        };
12764        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12765        let reap = std::thread::spawn(move || {
12766            tokio::runtime::Builder::new_current_thread()
12767                .enable_all()
12768                .build()
12769                .unwrap()
12770                .block_on(async {
12771                    let mut child =
12772                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12773                    health_restart_child(
12774                        &spec,
12775                        &runtime,
12776                        &supervisor.registry,
12777                        &supervisor.process_liveness,
12778                        &snapshot,
12779                        &mut child,
12780                        SupervisorHealthStatus::Failing,
12781                        Some("health probe did not answer"),
12782                        1,
12783                    )
12784                    .await
12785                    .unwrap();
12786                    assert!(child.is_none());
12787                });
12788        });
12789        // The loopback barrier acknowledges entry into terminal recording by
12790        // health_restart_child's drain, then waits for the test's release byte.
12791        // Status and history are observed while that writer is held, not after
12792        // an elapsed-time guess about when the health action has completed.
12793        let mut reached = [0];
12794        barrier.read_exact(&mut reached).unwrap();
12795        assert_eq!(&reached, b"R");
12796        let before_status = module.status().unwrap();
12797        let before_history = module.terminal_history();
12798        barrier.write_all(b"X").unwrap();
12799        reap.join().unwrap();
12800        assert!(before_history.entries.is_empty());
12801        assert_ne!(
12802            before_status.state,
12803            ModuleState::Failed,
12804            "Failed was visible before its terminal record could be written"
12805        );
12806        assert!(
12807            before_status.process_alive,
12808            "process death was visible before its terminal record could be written"
12809        );
12810        let failed = module.status().unwrap();
12811        assert_eq!(failed.state, ModuleState::Failed);
12812        assert!(!failed.process_alive);
12813        let history = module.terminal_history();
12814        assert_eq!(history.entries.len(), 1);
12815        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12816        assert_eq!(
12817            history.entries[0].disposition_detail.as_deref(),
12818            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12819        );
12820    }
12821
12822    #[test]
12823    fn observed_exits_publish_process_death_after_terminal_history() {
12824        use std::io::{Read, Write};
12825        for (kind, enabled, shutdown, wait_error, expected_state) in [
12826            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12827            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12828            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12829            (
12830                ExitKind::DeliberateSeverance,
12831                true,
12832                false,
12833                false,
12834                ModuleState::Restarting,
12835            ),
12836            #[cfg(unix)]
12837            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12838            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12839        ] {
12840            let supervisor = Supervisor::new_for_test(
12841                Arc::new(Registry::default()),
12842                RestartPolicy::new(1, Duration::ZERO),
12843            );
12844            let runtime = supervisor.runtime_config();
12845            #[cfg(unix)]
12846            if shutdown {
12847                runtime.child_roster.close();
12848            }
12849            #[cfg(not(unix))]
12850            let _ = shutdown;
12851            let spec = windowed_crash_spec("exit-publication-order");
12852            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12853                ModuleState::Running,
12854                enabled,
12855            )));
12856            lock_snapshot(&snapshot).unwrap().process_alive = true;
12857            let reaped_snapshot = snapshot.clone();
12858            let history = runtime.terminal_ring.clone();
12859            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12860            let reap = std::thread::spawn(move || {
12861                if wait_error {
12862                    super::fail_after_wait_error(
12863                        &spec.module_id,
12864                        &reaped_snapshot,
12865                        &runtime.terminal_ring,
12866                        &runtime.spawn_events,
12867                        "wait failed".into(),
12868                        &runtime.failure_context(true),
12869                    );
12870                } else {
12871                    tokio::runtime::Builder::new_current_thread()
12872                        .enable_all()
12873                        .build()
12874                        .unwrap()
12875                        .block_on(on_child_exit(
12876                            &spec,
12877                            runtime.restart_policy,
12878                            &supervisor.registry,
12879                            &reaped_snapshot,
12880                            &runtime.terminal_ring,
12881                            &runtime.spawn_events,
12882                            &runtime.child_roster,
12883                            ExitReport {
12884                                kind,
12885                                code: Some(0),
12886                                signal: None,
12887                                at_ms: 1,
12888                            },
12889                            true,
12890                        ));
12891                }
12892            });
12893            let mut reached = [0];
12894            barrier.read_exact(&mut reached).unwrap();
12895            assert_eq!(&reached, b"R");
12896            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12897            let before_history = history.lock().unwrap().snapshot();
12898            barrier.write_all(b"X").unwrap();
12899            reap.join().unwrap();
12900            assert!(
12901                before_alive,
12902                "{kind:?}: death was published before recording"
12903            );
12904            assert!(before_history.entries.is_empty());
12905            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12906            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12907            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12908        }
12909    }
12910
12911    #[tokio::test]
12912    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12913        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12914        let path = dir.join("terminals.jsonl");
12915        std::fs::create_dir(&path).unwrap();
12916        let supervisor = Supervisor::new_for_test(
12917            Arc::new(Registry::default()),
12918            RestartPolicy::new(0, Duration::ZERO),
12919        )
12920        .with_terminal_journal(path, "health-budget-journal-failure".into());
12921        let runtime = supervisor.runtime_config();
12922        let spec = ModuleSpec {
12923            program: fake_aft_stub_path(),
12924            protocol: ModuleProtocol::None,
12925            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12926            ..windowed_crash_spec("health-budget-journal-failure")
12927        };
12928        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12929        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12930        health_restart_child(
12931            &spec,
12932            &runtime,
12933            &supervisor.registry,
12934            &supervisor.process_liveness,
12935            &snapshot,
12936            &mut child,
12937            SupervisorHealthStatus::Failing,
12938            None,
12939            1,
12940        )
12941        .await
12942        .unwrap();
12943        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12944        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12945        let history = runtime
12946            .terminal_ring
12947            .lock()
12948            .unwrap()
12949            .durable_history(&spec.module_id);
12950        assert!(history.journal_write_failures > 0);
12951        assert_eq!(history.entries.len(), 1);
12952        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12953    }
12954
12955    #[test]
12956    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12957        let supervisor = Supervisor::new_for_test(
12958            Arc::new(Registry::default()),
12959            RestartPolicy::new(0, Duration::ZERO),
12960        );
12961        let runtime = supervisor.runtime_config();
12962        let spec = ModuleSpec {
12963            program: fake_aft_stub_path(),
12964            protocol: ModuleProtocol::None,
12965            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12966            ..windowed_crash_spec("health-budget-recording-panic")
12967        };
12968        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12969        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12970            tokio::runtime::Builder::new_current_thread()
12971                .enable_all()
12972                .build()
12973                .unwrap()
12974                .block_on(async {
12975                    let mut child =
12976                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12977                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12978                    health_restart_child(
12979                        &spec,
12980                        &runtime,
12981                        &supervisor.registry,
12982                        &supervisor.process_liveness,
12983                        &snapshot,
12984                        &mut child,
12985                        SupervisorHealthStatus::Failing,
12986                        None,
12987                        1,
12988                    )
12989                    .await
12990                    .unwrap();
12991                });
12992        }));
12993        let panic = result.expect_err("recording must still unwind");
12994        assert_eq!(
12995            panic.downcast_ref::<String>().map(String::as_str),
12996            Some("spawn event sequence exhausted")
12997        );
12998        let state = lock_snapshot(&snapshot).unwrap();
12999        assert_eq!(state.state, ModuleState::Failed);
13000        assert!(!state.process_alive);
13001        assert!(state.last_exit.is_some());
13002    }
13003
13004    #[tokio::test]
13005    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13006        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13007        let path = dir.join("terminals.jsonl");
13008        std::fs::create_dir(&path).unwrap();
13009        let supervisor = Supervisor::new_for_test(
13010            Arc::new(Registry::default()),
13011            RestartPolicy::new(0, Duration::ZERO),
13012        )
13013        .with_terminal_journal(path, "budget-journal-failure".into());
13014        let runtime = supervisor.runtime_config();
13015        let spec = windowed_crash_spec("budget-journal-failure");
13016        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13017        assert!(matches!(
13018            on_child_exit(
13019                &spec,
13020                runtime.restart_policy,
13021                &supervisor.registry,
13022                &snapshot,
13023                &runtime.terminal_ring,
13024                &runtime.spawn_events,
13025                &runtime.child_roster,
13026                crash_exit_report(1),
13027                true,
13028            )
13029            .await,
13030            NextAction::Stop { .. }
13031        ));
13032        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13033        let history = runtime
13034            .terminal_ring
13035            .lock()
13036            .unwrap()
13037            .durable_history(&spec.module_id);
13038        assert!(history.journal_write_failures > 0);
13039        assert_eq!(history.entries.len(), 1);
13040        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13041    }
13042
13043    #[test]
13044    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13045        let supervisor = Supervisor::new_for_test(
13046            Arc::new(Registry::default()),
13047            RestartPolicy::new(0, Duration::ZERO),
13048        );
13049        let runtime = supervisor.runtime_config();
13050        let spec = windowed_crash_spec("budget-recording-panic");
13051        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13052        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13053        // Exhausting the event sequence makes emit_exited panic before the
13054        // terminal write, exercising publication on the recording unwind.
13055        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13056        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13057            tokio::runtime::Builder::new_current_thread()
13058                .enable_all()
13059                .build()
13060                .unwrap()
13061                .block_on(on_child_exit(
13062                    &spec,
13063                    runtime.restart_policy,
13064                    &supervisor.registry,
13065                    &snapshot,
13066                    &runtime.terminal_ring,
13067                    &runtime.spawn_events,
13068                    &runtime.child_roster,
13069                    crash_exit_report(1),
13070                    true,
13071                ))
13072        }));
13073        let panic = result.err().expect("recording must still unwind");
13074        assert_eq!(
13075            panic.downcast_ref::<String>().map(String::as_str),
13076            Some("spawn event sequence exhausted")
13077        );
13078        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13079    }
13080
13081    /// A stop the supervisor itself requests still stops a protocol-none
13082    /// module, even though the child answers the SIGTERM with exit 0.
13083    #[cfg(unix)]
13084    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13085    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13086        for disable in [false, true] {
13087            let label = if disable {
13088                "none-requested-disable"
13089            } else {
13090                "none-requested-stop"
13091            };
13092            let dir = cortexkit_test_support::ScratchDir::new(label);
13093            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13094            let supervisor = Supervisor::new_for_test(
13095                Arc::new(Registry::default()),
13096                RestartPolicy::new(3, Duration::ZERO),
13097            );
13098            let module = supervisor.spawn(spec).unwrap();
13099            wait_for_file(&ready).await;
13100
13101            if disable {
13102                module.set_enabled(false).await.unwrap();
13103            } else {
13104                module.stop().await.unwrap();
13105            }
13106            assert!(
13107                marker.exists(),
13108                "{label}: the child must have left through its SIGTERM handler with exit 0"
13109            );
13110
13111            // Long enough for a zero-backoff respawn to have happened if the
13112            // exit had been treated as a crash.
13113            sleep(Duration::from_millis(500)).await;
13114            let status = module.status().unwrap();
13115            let expected = if disable {
13116                ModuleState::Disabled
13117            } else {
13118                ModuleState::Stopped
13119            };
13120            assert_eq!(status.state, expected, "{label}");
13121            assert_eq!(
13122                status.spawn_generation, 1,
13123                "{label}: respawned after a requested stop"
13124            );
13125            let history = module.terminal_history();
13126            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13127            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13128            assert_ne!(
13129                history.entries[0].disposition,
13130                TerminalDisposition::Restarting,
13131                "{label}"
13132            );
13133        }
13134    }
13135
13136    /// A subc-wire module that exits 0 on its own is still a stop: the
13137    /// protocol-none rule must not reach it.
13138    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13139    async fn subc_wire_clean_exit_is_still_a_stop() {
13140        let supervisor = Supervisor::new_for_test(
13141            Arc::new(Registry::default()),
13142            RestartPolicy::new(3, Duration::ZERO),
13143        );
13144        let module = supervisor
13145            .spawn(ModuleSpec {
13146                module_id: "wire-clean-exit".to_string(),
13147                program: fake_aft_stub_path(),
13148                args: Vec::new(),
13149                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13150                reserved: false,
13151                reserved_prefixes: Vec::new(),
13152                protocol: ModuleProtocol::Subc,
13153                overlap: Default::default(),
13154            })
13155            .unwrap();
13156
13157        let deadline = Instant::now() + Duration::from_secs(10);
13158        while module.terminal_history().entries.is_empty() {
13159            assert!(Instant::now() < deadline, "module never exited");
13160            sleep(Duration::from_millis(10)).await;
13161        }
13162        // Long enough for a zero-backoff respawn to have happened.
13163        sleep(Duration::from_millis(500)).await;
13164        let status = module.status().unwrap();
13165        assert_eq!(status.state, ModuleState::Stopped);
13166        assert_eq!(status.spawn_generation, 1);
13167        let history = module.terminal_history();
13168        assert_eq!(history.entries.len(), 1, "{history:?}");
13169        assert_eq!(history.entries[0].exit_code, Some(0));
13170        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13171    }
13172
13173    #[cfg(unix)]
13174    #[tokio::test]
13175    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13176        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13177        let record = dir.join("live-children.json");
13178        let supervisor = Supervisor::new_for_test(
13179            Arc::new(Registry::default()),
13180            RestartPolicy::new(0, Duration::ZERO),
13181        );
13182        let mut runtime = supervisor.runtime_config();
13183        runtime.child_roster.record_to(record.clone());
13184        let gate = Arc::new(super::ReloadExitRecordGate::default());
13185        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13186        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13187        let spec = ModuleSpec {
13188            module_id: "reload-exit-roster".into(),
13189            program: fake_aft_stub_path(),
13190            args: Vec::new(),
13191            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13192            reserved: false,
13193            reserved_prefixes: Vec::new(),
13194            protocol: ModuleProtocol::Subc,
13195            overlap: Default::default(),
13196        };
13197        let mut child = None;
13198        let reload = super::finish_reload_child(
13199            &spec,
13200            &runtime,
13201            &supervisor.registry,
13202            &supervisor.process_liveness,
13203            &snapshot,
13204            &mut child,
13205        );
13206        tokio::pin!(reload);
13207        tokio::select! {
13208            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13209            _ = gate.reached.notified() => {}
13210        }
13211        assert!(runtime
13212            .terminal_ring
13213            .lock()
13214            .unwrap()
13215            .snapshot()
13216            .entries
13217            .is_empty());
13218        assert_eq!(
13219            crate::live_children::read_record(&record).unwrap().len(),
13220            1,
13221            "shutdown must still wait for the reaped child until its terminal record exists"
13222        );
13223        runtime.child_roster.close();
13224        gate.resume.notify_one();
13225        assert!(reload.await.is_err());
13226        assert!(crate::live_children::read_record(&record)
13227            .unwrap()
13228            .is_empty());
13229        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13230        assert_eq!(history.entries.len(), 1);
13231        assert_eq!(
13232            history.entries[0].disposition,
13233            TerminalDisposition::DaemonShutdown
13234        );
13235    }
13236
13237    /// Each restart-producing arm has its own state transition. Keeping their
13238    /// lifetime count assertions adjacent prevents a later new arm from silently
13239    /// spending budget without recording the historical restart.
13240    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13241    async fn every_restart_increment_path_advances_lifetime_count() {
13242        let supervisor = Supervisor::new_for_test(
13243            Arc::new(Registry::default()),
13244            RestartPolicy::new(1, Duration::ZERO),
13245        );
13246        let runtime = supervisor.runtime_config();
13247        let spec = ModuleSpec {
13248            module_id: "lifetime-increment-path".to_string(),
13249            program: PathBuf::from("/unused/lifetime-increment-path"),
13250            args: Vec::new(),
13251            env: Vec::new(),
13252            reserved: false,
13253            reserved_prefixes: Vec::new(),
13254            protocol: ModuleProtocol::Subc,
13255            overlap: Default::default(),
13256        };
13257
13258        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13259        assert!(matches!(
13260            on_child_exit(
13261                &spec,
13262                runtime.restart_policy,
13263                &supervisor.registry,
13264                &crash_snapshot,
13265                &runtime.terminal_ring,
13266                &runtime.spawn_events,
13267                &runtime.child_roster,
13268                ExitReport {
13269                    kind: ExitKind::Crash,
13270                    code: Some(1),
13271                    signal: None,
13272                    at_ms: 1,
13273                },
13274                true,
13275            )
13276            .await,
13277            NextAction::Restart { schedule: _ }
13278        ));
13279        let (crash_restarts, crash_lifetime) = {
13280            let state = lock_snapshot(&crash_snapshot).unwrap();
13281            (state.crash_restarts.len(), state.lifetime_restarts)
13282        };
13283        assert_eq!(crash_restarts, 1);
13284        assert_eq!(crash_lifetime, 1);
13285
13286        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13287        let mut health_child = None;
13288        assert!(matches!(
13289            health_restart_child(
13290                &spec,
13291                &runtime,
13292                &supervisor.registry,
13293                &supervisor.process_liveness,
13294                &health_snapshot,
13295                &mut health_child,
13296                SupervisorHealthStatus::Failing,
13297                None,
13298                2,
13299            )
13300            .await,
13301            Ok(())
13302        ));
13303        assert!(health_child.is_none());
13304        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13305        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13306        let (health_restarts, health_lifetime) = {
13307            let state = lock_snapshot(&health_snapshot).unwrap();
13308            (state.crash_restarts.len(), state.lifetime_restarts)
13309        };
13310        assert_eq!(health_restarts, 1);
13311        assert_eq!(health_lifetime, 1);
13312
13313        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13314        let mut reload_child = None;
13315        assert!(matches!(
13316            handle_reload_spawn_failure(
13317                &spec,
13318                &runtime,
13319                &supervisor.process_liveness,
13320                &reload_snapshot,
13321                &mut reload_child,
13322                "forced reload spawn failure".to_string(),
13323            )
13324            .await,
13325            Err(SuperviseError::ReloadFailed { .. })
13326        ));
13327        let (reload_restarts, reload_lifetime) = {
13328            let state = lock_snapshot(&reload_snapshot).unwrap();
13329            (state.crash_restarts.len(), state.lifetime_restarts)
13330        };
13331        assert_eq!(reload_restarts, 1);
13332        assert_eq!(reload_lifetime, 1);
13333    }
13334
13335    #[tokio::test]
13336    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13337        let supervisor = Supervisor::new_for_test(
13338            Arc::new(Registry::default()),
13339            RestartPolicy::new(3, Duration::ZERO),
13340        );
13341        let runtime = supervisor.runtime_config();
13342        let spec = ModuleSpec {
13343            module_id: "deliberately-severed".to_string(),
13344            program: PathBuf::from("/unused/deliberately-severed"),
13345            args: Vec::new(),
13346            env: Vec::new(),
13347            reserved: false,
13348            reserved_prefixes: Vec::new(),
13349            protocol: ModuleProtocol::Subc,
13350            overlap: Default::default(),
13351        };
13352        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13353        let process = ProcessIdentity {
13354            pid: 41,
13355            start_time: 101,
13356        };
13357        record_deliberate_severance(&snapshot, process).unwrap();
13358        let exit_report = apply_deliberate_severance_marker(
13359            &snapshot,
13360            Some(process),
13361            ExitReport {
13362                kind: ExitKind::Crash,
13363                code: Some(1),
13364                signal: None,
13365                at_ms: 1,
13366            },
13367        );
13368        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13369
13370        assert!(matches!(
13371            on_child_exit(
13372                &spec,
13373                runtime.restart_policy,
13374                &supervisor.registry,
13375                &snapshot,
13376                &runtime.terminal_ring,
13377                &runtime.spawn_events,
13378                &runtime.child_roster,
13379                exit_report,
13380                true,
13381            )
13382            .await,
13383            NextAction::Restart { schedule: _ }
13384        ));
13385        let state = lock_snapshot(&snapshot).unwrap();
13386        assert_eq!(state.lifetime_restarts, 1);
13387        assert_eq!(state.crash_restarts.len(), 0);
13388    }
13389
13390    #[tokio::test]
13391    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13392        let supervisor = Supervisor::new_for_test(
13393            Arc::new(Registry::default()),
13394            RestartPolicy::new(3, Duration::ZERO),
13395        );
13396        let runtime = supervisor.runtime_config();
13397        let spec = ModuleSpec {
13398            module_id: "genuine-crash".to_string(),
13399            program: PathBuf::from("/unused/genuine-crash"),
13400            args: Vec::new(),
13401            env: Vec::new(),
13402            reserved: false,
13403            reserved_prefixes: Vec::new(),
13404            protocol: ModuleProtocol::Subc,
13405            overlap: Default::default(),
13406        };
13407        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13408
13409        assert!(matches!(
13410            on_child_exit(
13411                &spec,
13412                runtime.restart_policy,
13413                &supervisor.registry,
13414                &snapshot,
13415                &runtime.terminal_ring,
13416                &runtime.spawn_events,
13417                &runtime.child_roster,
13418                ExitReport {
13419                    kind: ExitKind::Crash,
13420                    code: Some(1),
13421                    signal: None,
13422                    at_ms: 1,
13423                },
13424                true,
13425            )
13426            .await,
13427            NextAction::Restart { schedule: _ }
13428        ));
13429        let state = lock_snapshot(&snapshot).unwrap();
13430        assert_eq!(state.lifetime_restarts, 1);
13431        assert_eq!(state.crash_restarts.len(), 1);
13432    }
13433
13434    fn crash_exit_report(at_ms: u64) -> ExitReport {
13435        ExitReport {
13436            kind: ExitKind::Crash,
13437            code: Some(1),
13438            signal: None,
13439            at_ms,
13440        }
13441    }
13442
13443    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13444        ModuleSpec {
13445            module_id: module_id.to_string(),
13446            program: PathBuf::from("/unused").join(module_id),
13447            args: Vec::new(),
13448            env: Vec::new(),
13449            reserved: false,
13450            reserved_prefixes: Vec::new(),
13451            protocol: ModuleProtocol::Subc,
13452            overlap: Default::default(),
13453        }
13454    }
13455
13456    /// A real crash loop still stops. Three crashes with nothing aging out spend
13457    /// a budget of two and the third respawn is refused, and both surfaces an
13458    /// operator has -- the log line and the retained terminal record -- name the
13459    /// window rather than only the cap, because `max_restarts=2` alone is what
13460    /// this budget used to mean.
13461    #[tokio::test]
13462    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13463        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13464        let supervisor = Supervisor::new_for_test(
13465            Arc::new(Registry::default()),
13466            RestartPolicy::new(2, Duration::ZERO),
13467        );
13468        let runtime = supervisor.runtime_config();
13469        let spec = windowed_crash_spec("crash-loop-in-window");
13470        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13471
13472        for attempt in 1..=2 {
13473            assert!(
13474                matches!(
13475                    on_child_exit(
13476                        &spec,
13477                        runtime.restart_policy,
13478                        &supervisor.registry,
13479                        &snapshot,
13480                        &runtime.terminal_ring,
13481                        &runtime.spawn_events,
13482                        &runtime.child_roster,
13483                        crash_exit_report(attempt),
13484                        true,
13485                    )
13486                    .await,
13487                    NextAction::Restart { schedule: _ }
13488                ),
13489                "crash {attempt} is inside the budget and must respawn"
13490            );
13491        }
13492
13493        assert!(matches!(
13494            on_child_exit(
13495                &spec,
13496                runtime.restart_policy,
13497                &supervisor.registry,
13498                &snapshot,
13499                &runtime.terminal_ring,
13500                &runtime.spawn_events,
13501                &runtime.child_roster,
13502                crash_exit_report(3),
13503                true,
13504            )
13505            .await,
13506            NextAction::Stop { .. }
13507        ));
13508
13509        {
13510            let state = lock_snapshot(&snapshot).unwrap();
13511            assert_eq!(state.state, ModuleState::Failed);
13512            assert_eq!(state.crash_restarts.len(), 2);
13513            assert_eq!(state.lifetime_restarts, 2);
13514        }
13515
13516        let history = runtime
13517            .terminal_ring
13518            .lock()
13519            .expect("terminal ring is not poisoned")
13520            .snapshot();
13521        let last = history
13522            .entries
13523            .last()
13524            .expect("the refused crash is retained");
13525        assert_eq!(last.disposition, TerminalDisposition::Failed);
13526        assert_eq!(
13527            last.disposition_detail.as_deref(),
13528            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13529        );
13530
13531        let captured = crate::router::test_log::captured_logs(&logs);
13532        assert!(
13533            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13534            "the stop must be logged with its window: {captured}"
13535        );
13536    }
13537
13538    /// The rate, stated as a test: three crashes where the first has aged past
13539    /// the window are two crashes as far as the budget is concerned, so the
13540    /// third respawn is allowed and the ring holds only the two recent ones.
13541    ///
13542    /// This is the case a lifetime counter got wrong -- and the case the daemon
13543    /// now hits routinely, since a module exits non-zero every time its
13544    /// connection to the daemon drops.
13545    #[tokio::test]
13546    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13547        let supervisor = Supervisor::new_for_test(
13548            Arc::new(Registry::default()),
13549            RestartPolicy::new(2, Duration::ZERO),
13550        );
13551        let runtime = supervisor.runtime_config();
13552        let spec = windowed_crash_spec("crash-across-windows");
13553        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13554
13555        for attempt in 1..=2 {
13556            assert!(matches!(
13557                on_child_exit(
13558                    &spec,
13559                    runtime.restart_policy,
13560                    &supervisor.registry,
13561                    &snapshot,
13562                    &runtime.terminal_ring,
13563                    &runtime.spawn_events,
13564                    &runtime.child_roster,
13565                    crash_exit_report(attempt),
13566                    true,
13567                )
13568                .await,
13569                NextAction::Restart { schedule: _ }
13570            ));
13571        }
13572
13573        // The oldest crash moves out of the window; nothing else about the
13574        // module changes.
13575        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13576            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13577        })
13578        .unwrap();
13579
13580        assert!(
13581            matches!(
13582                on_child_exit(
13583                    &spec,
13584                    runtime.restart_policy,
13585                    &supervisor.registry,
13586                    &snapshot,
13587                    &runtime.terminal_ring,
13588                    &runtime.spawn_events,
13589                    &runtime.child_roster,
13590                    crash_exit_report(3),
13591                    true,
13592                )
13593                .await,
13594                NextAction::Restart { schedule: _ }
13595            ),
13596            "a crash older than the window must not hold a budget slot"
13597        );
13598
13599        let state = lock_snapshot(&snapshot).unwrap();
13600        assert_eq!(state.state, ModuleState::Restarting);
13601        assert_eq!(
13602            state.crash_restarts.len(),
13603            2,
13604            "the aged instant is dropped and the new one takes its place"
13605        );
13606        assert_eq!(
13607            state.lifetime_restarts, 3,
13608            "the ledger counts every restart, including the ones the window forgot"
13609        );
13610    }
13611
13612    /// An operator restart hands the budget back whole, and the ledger keeps
13613    /// counting. Those are different questions -- "how close is this module to
13614    /// being stopped" and "how many times has it been replaced" -- and the
13615    /// operator action answers only the first.
13616    #[tokio::test]
13617    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13618        let supervisor = Supervisor::new_for_test(
13619            Arc::new(Registry::default()),
13620            RestartPolicy::new(2, Duration::ZERO),
13621        );
13622        let runtime = supervisor.runtime_config();
13623        let spec = windowed_crash_spec("operator-cleared-budget");
13624        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13625
13626        for attempt in 1..=2 {
13627            assert!(matches!(
13628                on_child_exit(
13629                    &spec,
13630                    runtime.restart_policy,
13631                    &supervisor.registry,
13632                    &snapshot,
13633                    &runtime.terminal_ring,
13634                    &runtime.spawn_events,
13635                    &runtime.child_roster,
13636                    crash_exit_report(attempt),
13637                    true,
13638                )
13639                .await,
13640                NextAction::Restart { schedule: _ }
13641            ));
13642        }
13643
13644        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13645        {
13646            let state = lock_snapshot(&snapshot).unwrap();
13647            assert!(
13648                state.crash_restarts.is_empty(),
13649                "an operator restart returns the full budget"
13650            );
13651            assert_eq!(
13652                state.lifetime_restarts, 2,
13653                "clearing the budget must not unmake the crashes"
13654            );
13655        }
13656
13657        assert!(
13658            matches!(
13659                on_child_exit(
13660                    &spec,
13661                    runtime.restart_policy,
13662                    &supervisor.registry,
13663                    &snapshot,
13664                    &runtime.terminal_ring,
13665                    &runtime.spawn_events,
13666                    &runtime.child_roster,
13667                    crash_exit_report(3),
13668                    true,
13669                )
13670                .await,
13671                NextAction::Restart { schedule: _ }
13672            ),
13673            "the cleared budget must be spendable again"
13674        );
13675        let state = lock_snapshot(&snapshot).unwrap();
13676        assert_eq!(state.crash_restarts.len(), 1);
13677        assert_eq!(state.lifetime_restarts, 3);
13678    }
13679
13680    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13681    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13682        let severed = ProcessIdentity {
13683            pid: 41,
13684            start_time: 101,
13685        };
13686        let successor = ProcessIdentity {
13687            pid: 41,
13688            start_time: 202,
13689        };
13690        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13691        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13692            state.pid = Some(successor.pid);
13693            state.process_start_time = Some(successor.start_time);
13694        })
13695        .unwrap();
13696        assert!(!module.record_deliberate_severance(severed).unwrap());
13697
13698        let exit_report = apply_deliberate_severance_marker(
13699            &module.inner.snapshot,
13700            Some(successor),
13701            ExitReport {
13702                kind: ExitKind::Crash,
13703                code: Some(1),
13704                signal: None,
13705                at_ms: 1,
13706            },
13707        );
13708
13709        assert_eq!(exit_report.kind, ExitKind::Crash);
13710    }
13711
13712    #[tokio::test]
13713    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13714        let registry = Registry::default();
13715        let supervisor = Supervisor::new_for_test(
13716            Arc::new(Registry::default()),
13717            RestartPolicy::new(3, Duration::ZERO),
13718        );
13719        let runtime = supervisor.runtime_config();
13720        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13721        let spec = ModuleSpec {
13722            module_id: "drain-deliberate-severance".to_string(),
13723            program: fake_aft_stub_path(),
13724            args: Vec::new(),
13725            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13726            reserved: false,
13727            reserved_prefixes: Vec::new(),
13728            protocol: ModuleProtocol::Subc,
13729            overlap: Default::default(),
13730        };
13731        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13732        let process = ProcessIdentity {
13733            pid: 41,
13734            start_time: 101,
13735        };
13736        child.process_identity = Some(process);
13737        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13738            state.pid = Some(process.pid);
13739            state.process_start_time = Some(process.start_time);
13740        })
13741        .unwrap();
13742        record_deliberate_severance(&snapshot, process).unwrap();
13743
13744        drain_child_to_state(
13745            &spec.module_id,
13746            spec.protocol,
13747            // The child exits on its own; no signal may change the exit this
13748            // test classifies.
13749            StopNotice::SentOverConnection,
13750            &registry,
13751            None,
13752            &snapshot,
13753            &runtime.terminal_ring,
13754            &runtime.spawn_events,
13755            child,
13756            Duration::from_secs(1),
13757            ModuleState::Stopped,
13758            Some(false),
13759            &super::FailureContext::operator(),
13760        )
13761        .await
13762        .unwrap();
13763
13764        let state = lock_snapshot(&snapshot).unwrap();
13765        assert_eq!(
13766            state.last_exit.as_ref().map(|exit| exit.kind),
13767            Some(ExitKind::DeliberateSeverance)
13768        );
13769        assert_eq!(state.lifetime_restarts, 1);
13770        assert_eq!(state.crash_restarts.len(), 0);
13771        drop(state);
13772        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13773        assert_eq!(
13774            history.entries[0].exit_kind,
13775            subc_control::TerminalExitKind::DeliberateSeverance
13776        );
13777    }
13778
13779    #[tokio::test]
13780    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13781        let registry = Registry::default();
13782        let supervisor = Supervisor::new_for_test(
13783            Arc::new(Registry::default()),
13784            RestartPolicy::new(3, Duration::ZERO),
13785        );
13786        let runtime = supervisor.runtime_config();
13787        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13788        let spec = ModuleSpec {
13789            module_id: "ordinary-drain".to_string(),
13790            program: fake_aft_stub_path(),
13791            args: Vec::new(),
13792            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13793            reserved: false,
13794            reserved_prefixes: Vec::new(),
13795            protocol: ModuleProtocol::Subc,
13796            overlap: Default::default(),
13797        };
13798        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13799
13800        drain_child_to_state(
13801            &spec.module_id,
13802            spec.protocol,
13803            // The child exits on its own; no signal may change the exit this
13804            // test classifies.
13805            StopNotice::SentOverConnection,
13806            &registry,
13807            None,
13808            &snapshot,
13809            &runtime.terminal_ring,
13810            &runtime.spawn_events,
13811            child,
13812            Duration::from_secs(1),
13813            ModuleState::Stopped,
13814            Some(false),
13815            &super::FailureContext::operator(),
13816        )
13817        .await
13818        .unwrap();
13819
13820        let state = lock_snapshot(&snapshot).unwrap();
13821        assert_eq!(
13822            state.last_exit.as_ref().map(|exit| exit.kind),
13823            Some(ExitKind::Crash)
13824        );
13825        assert_eq!(state.lifetime_restarts, 0);
13826        assert_eq!(state.crash_restarts.len(), 0);
13827    }
13828
13829    #[test]
13830    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13831        // The server's generic fatal-routing branch only knows that the
13832        // connection failed; it does not know that the daemon deliberately
13833        // initiated a process-killing severance. Keep this seam explicit so a
13834        // future connection error path cannot silently reintroduce the stale
13835        // exemption that mislabels a later genuine crash.
13836        assert!(!include_str!("server.rs")
13837            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13838    }
13839
13840    /// The `route.closed` `drained` value must be the quiescence wait's own
13841    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13842    /// measurement at all and `false` is the one honest constant. This is the exact
13843    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13844    /// on every return path, including the one that used to return early via `?`
13845    /// with `route.closing` already sent and no `route.closed` ever following.
13846    #[test]
13847    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13848        assert!(drained_after_quiescence_wait(&Ok(true)));
13849        assert!(!drained_after_quiescence_wait(&Ok(false)));
13850        assert!(!drained_after_quiescence_wait(&Err(
13851            SuperviseError::StatePoisoned { module_id: None }
13852        )));
13853    }
13854
13855    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13856    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13857    /// already reaped out-of-band) still leaves a terminal record rather than none
13858    /// at all. Triggering the real `wait()` I/O error from an integration test would
13859    /// need a genuine already-reaped-child race, which is OS-specific and not
13860    /// something this suite attempts elsewhere; this test instead verifies the
13861    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13862    /// the call site itself is verified by inspection to sit in that exact arm.
13863    #[test]
13864    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13865        let ring = Arc::new(Mutex::new(TerminalRing::new(
13866            TerminalRingConfig::default(),
13867            0,
13868        )));
13869        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13870
13871        let snapshot = ring.lock().unwrap().snapshot();
13872        assert_eq!(snapshot.entries.len(), 1);
13873        let entry = &snapshot.entries[0];
13874        assert_eq!(entry.exit_code, None);
13875        assert_eq!(entry.exit_signal, None);
13876        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13877    }
13878
13879    #[test]
13880    fn wait_error_exit_path_preserves_spawn_event_density() {
13881        let feed = super::SpawnEventFeed::default();
13882        feed.configure_incarnation("wait-error-density".to_string());
13883        feed.emit_spawned("wait-error", 41, 1);
13884        let ring = Arc::new(Mutex::new(TerminalRing::new(
13885            TerminalRingConfig::default(),
13886            0,
13887        )));
13888
13889        record_wait_error_terminal("wait-error", &ring, &feed);
13890        feed.emit_spawned("after-wait-error", 42, 2);
13891
13892        let state = feed.0.lock().unwrap();
13893        let sequences = state
13894            .events
13895            .iter()
13896            .map(|event| event.cursor.seq)
13897            .collect::<Vec<_>>();
13898        assert_eq!(sequences, vec![1, 2, 3]);
13899        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13900        assert_eq!(state.events[1].exit_code, None);
13901        assert_eq!(state.events[1].exit_signal, None);
13902    }
13903
13904    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13905    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13906    /// not a clean exit it never actually observed.
13907    #[test]
13908    fn wait_error_exit_report_is_classified_as_a_crash() {
13909        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13910    }
13911}
13912
13913#[cfg(all(test, unix))]
13914mod restart_ordering_tests {
13915    use super::*;
13916    use std::io::Write;
13917    use tokio::io::AsyncReadExt;
13918
13919    #[test]
13920    #[ignore = "child process fixture"]
13921    fn serving_child() {
13922        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13923            return;
13924        };
13925        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13926        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13927        for stream in listener.incoming() {
13928            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13929        }
13930    }
13931
13932    async fn restart_before_publication(requests: usize) {
13933        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13934        let address_file = home.path().join("address");
13935        let spec = ModuleSpec {
13936            module_id: "restart-serving-child".into(),
13937            program: std::env::current_exe().unwrap(),
13938            args: vec![
13939                "--exact".into(),
13940                "supervise::restart_ordering_tests::serving_child".into(),
13941                "--ignored".into(),
13942                "--nocapture".into(),
13943            ],
13944            env: vec![(
13945                "RESTART_TEST_ADDRESS_FILE".into(),
13946                address_file.display().to_string(),
13947            )],
13948            reserved: false,
13949            reserved_prefixes: vec![],
13950            protocol: ModuleProtocol::None,
13951            overlap: Default::default(),
13952        };
13953        let mut supervisor =
13954            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13955        let (spawned_tx, spawned_rx) = oneshot::channel();
13956        let spawned_tx = Mutex::new(Some(spawned_tx));
13957        let (release_tx, release_rx) = std::sync::mpsc::channel();
13958        let release_rx = Mutex::new(release_rx);
13959        // Hold publication, not the child: it must answer a real request before
13960        // the restart enters the queue, even though status still has generation 0.
13961        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13962            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13963            release_rx
13964                .lock()
13965                .unwrap()
13966                .recv_timeout(Duration::from_secs(20))
13967                .unwrap();
13968        })));
13969        let registered = supervisor
13970            .register_configured_with_health(
13971                spec.clone(),
13972                true,
13973                HealthConfig::default(),
13974                None,
13975                RestartPolicy::new(3, Duration::ZERO),
13976                DEFAULT_REGISTER_TIMEOUT,
13977            )
13978            .unwrap();
13979        let module = registered.module.clone();
13980        let startup = tokio::task::spawn_blocking(move || {
13981            supervisor.start_registered_configured(registered, None, true)
13982        });
13983        spawned_rx.await.unwrap();
13984        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13985            let address = loop {
13986                if let Ok(address) = std::fs::read_to_string(&address_file) {
13987                    if !address.is_empty() {
13988                        break address;
13989                    }
13990                }
13991                tokio::time::sleep(Duration::from_millis(1)).await;
13992            };
13993            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13994            let mut reply = String::new();
13995            stream.read_to_string(&mut reply).await.unwrap();
13996            reply.trim().parse::<u32>().unwrap()
13997        })
13998        .await
13999        .expect("child must serve before generation publication");
14000        assert_eq!(
14001            lock_snapshot(&module.inner.snapshot)
14002                .unwrap()
14003                .spawn_generation,
14004            0
14005        );
14006        let mut replies = Vec::new();
14007        for _ in 0..requests {
14008            let mut restart = Box::pin(module.restart(Some(0)));
14009            // Poll the public API through queue admission, leaving only its ACK
14010            // pending. The monitor cannot consume the command until released.
14011            std::future::poll_fn(|cx| {
14012                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14013                std::task::Poll::Ready(())
14014            })
14015            .await;
14016            replies.push(restart);
14017        }
14018        release_tx.send(()).unwrap();
14019        startup.await.unwrap().unwrap();
14020        for reply in replies {
14021            reply.await.unwrap();
14022        }
14023        // Restart schedules its replacement after completing the handler. Wait
14024        // for that separate actor turn before placing the command barrier.
14025        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14026            loop {
14027                if lock_snapshot(&module.inner.snapshot)
14028                    .unwrap()
14029                    .spawn_generation
14030                    > 1
14031                {
14032                    break;
14033                }
14034                tokio::time::sleep(Duration::from_millis(1)).await;
14035            }
14036        })
14037        .await;
14038        // Commands are handled serially. The update-spec reply proves every
14039        // restart handler has finished before we inspect the snapshot; restart's
14040        // own ACK only acknowledges initiation, not replacement completion.
14041        module.update_spec_for_test(spec).await.unwrap();
14042        let (generation, pid) = {
14043            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14044            (state.spawn_generation, state.pid)
14045        };
14046        module.stop().await.unwrap();
14047        replacement.expect("restart must publish a replacement for the already-serving child");
14048        assert_ne!(
14049            pid,
14050            Some(old_pid),
14051            "a serving child predating the restart must be replaced"
14052        );
14053        assert_eq!(generation, 2, "queued restarts must share one replacement");
14054    }
14055
14056    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14057    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14058        restart_before_publication(1).await;
14059    }
14060
14061    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14062    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14063        restart_before_publication(2).await;
14064    }
14065
14066    #[tokio::test]
14067    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14068        let supervisor =
14069            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14070        let registered = supervisor
14071            .register_configured_with_health(
14072                ModuleSpec {
14073                    module_id: "queued-before-replacement".into(),
14074                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14075                    // macOS a module whose running image is not the configured
14076                    // program is refused once its exec is confirmed (see
14077                    // `record_write_stall_tests::sleeping_spec`).
14078                    program: "/bin/sleep".into(),
14079                    args: vec!["600".into()],
14080                    env: vec![],
14081                    reserved: false,
14082                    reserved_prefixes: vec![],
14083                    protocol: ModuleProtocol::None,
14084                    overlap: Default::default(),
14085                },
14086                true,
14087                HealthConfig::default(),
14088                None,
14089                RestartPolicy::default(),
14090                DEFAULT_REGISTER_TIMEOUT,
14091            )
14092            .unwrap();
14093        let RegisteredConfiguredModule {
14094            mut spec,
14095            mut runtime,
14096            module,
14097            mut commands,
14098        } = registered;
14099        let snapshot = &module.inner.snapshot;
14100        let mut restart = Box::pin(module.restart(Some(0)));
14101        std::future::poll_fn(|cx| {
14102            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14103            std::task::Poll::Ready(())
14104        })
14105        .await;
14106        // Drive the actor explicitly so a replacement is definitely spawned
14107        // after receipt, but before this queued command is dequeued.
14108        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14109        let replacement_pid = child.as_ref().unwrap().id();
14110        let command = commands.recv().await.unwrap();
14111        assert!(
14112            handle_supervisor_command(
14113                command,
14114                &mut spec,
14115                &mut runtime,
14116                &supervisor.registry,
14117                &supervisor.process_liveness,
14118                snapshot,
14119                &mut child,
14120                &mut commands,
14121                &mut VecDeque::new(),
14122            )
14123            .await
14124        );
14125        restart.await.unwrap();
14126        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14127        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14128        if let Some(mut child) = child {
14129            child.start_kill().unwrap();
14130            child.wait().await.unwrap();
14131        }
14132        assert_eq!(
14133            kept_pid, replacement_pid,
14134            "a newer replacement must survive"
14135        );
14136        assert!(
14137            !pending,
14138            "a satisfied restart must not schedule another spawn"
14139        );
14140    }
14141}
14142
14143#[cfg(all(test, unix))]
14144mod health_event_tests {
14145    use super::*;
14146    use cortexkit_test_support::ScratchDir;
14147    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14148
14149    struct Actor {
14150        module: SupervisedModule,
14151        registry: Arc<Registry>,
14152        forwarding: Arc<ForwardingTable>,
14153        spec: ModuleSpec,
14154        health: HealthConfig,
14155        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14156        roster: ChildRoster,
14157        _home: ScratchDir,
14158    }
14159
14160    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14161        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14162        // for an observed condition, rather than letting paused time auto-advance
14163        // deadlines or assuming a fixed number of yields finishes the work.
14164        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14165        let tick = Instant::now();
14166        loop {
14167            if let Some(value) = observe() {
14168                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14169                return value;
14170            }
14171            assert!(
14172                std::time::Instant::now() < deadline,
14173                "timed out waiting for {reason}"
14174            );
14175            tokio::task::yield_now().await;
14176        }
14177    }
14178
14179    impl Actor {
14180        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14181            Self::start_with_policy(
14182                protocol,
14183                cadence,
14184                DEFAULT_REGISTER_TIMEOUT,
14185                RestartPolicy::new(3, Duration::ZERO),
14186            )
14187            .await
14188        }
14189
14190        async fn start_with_policy(
14191            protocol: ModuleProtocol,
14192            cadence: Duration,
14193            register_timeout: Duration,
14194            policy: RestartPolicy,
14195        ) -> Self {
14196            let home = ScratchDir::new("health-events");
14197            let registry = Arc::new(Registry::default());
14198            let forwarding = Arc::new(ForwardingTable::default());
14199            let health = HealthConfig {
14200                cadence,
14201                deadline: Duration::from_secs(3600),
14202                ..HealthConfig::default()
14203            };
14204            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14205                .with_forwarding(forwarding.clone())
14206                .with_health_config(health.clone());
14207            let spec = ModuleSpec {
14208                module_id: "event-health-child".into(),
14209                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14210                // the privacy trampoline refuses a module whose running image
14211                // is not the configured program, and `start` below waits for
14212                // the process to be confirmed (`reported_pid`). The shell form
14213                // passed only while the image check ran before the shell had
14214                // replaced itself with `sleep` (see
14215                // `record_write_stall_tests::sleeping_spec`).
14216                program: PathBuf::from("/bin/sleep"),
14217                // The process stays alive but has no real bus peer. Registrations
14218                // below exercise the same registry writes as accepted HELLOs.
14219                args: vec!["600".into()],
14220                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14221                    .into_iter()
14222                    .map(|key| (key.into(), home.path().display().to_string()))
14223                    .collect(),
14224                reserved: false,
14225                reserved_prefixes: vec![],
14226                protocol,
14227                overlap: Default::default(),
14228            };
14229            let module = supervisor
14230                .supervise_configured_requested(
14231                    spec.clone(),
14232                    true,
14233                    health.clone(),
14234                    Some(0),
14235                    policy,
14236                    register_timeout,
14237                    None,
14238                )
14239                .unwrap();
14240            let (_, rx) = mpsc::channel(8);
14241            let actor = Self {
14242                module,
14243                registry,
14244                forwarding,
14245                spec,
14246                health,
14247                rx,
14248                roster: supervisor.child_roster.clone(),
14249                _home: home,
14250            };
14251            actor
14252                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14253                .await;
14254            actor
14255        }
14256
14257        async fn expire_registration_wait(&self, remaining: Duration) {
14258            tokio::time::advance(remaining).await;
14259            wait_for("registration timeout TERM grace", || {
14260                lock_snapshot(&self.module.inner.snapshot)
14261                    .unwrap()
14262                    .actor_select
14263                    .is_none()
14264                    .then_some(())
14265            })
14266            .await;
14267            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14268        }
14269
14270        fn turns(&self) -> u64 {
14271            lock_snapshot(&self.module.inner.snapshot)
14272                .unwrap()
14273                .actor_turns
14274        }
14275
14276        async fn wait_for_select(
14277            &self,
14278            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14279        ) -> ActorSelectCheckpoint {
14280            wait_for("actor select checkpoint", || {
14281                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14282                state.actor_select.filter(|checkpoint| {
14283                    checkpoint.generation == state.spawn_generation
14284                        && state.state == ModuleState::Running
14285                        && state.process_alive
14286                        && state.reported_pid().is_some()
14287                        && expected(checkpoint)
14288                })
14289            })
14290            .await
14291        }
14292
14293        fn hello(&mut self, connection: u64, health: bool) {
14294            let manifest =
14295                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14296                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14297                    .build();
14298            let connection = ConnectionId::new(connection);
14299            let (tx, rx) = mpsc::channel(8);
14300            self.rx = rx;
14301            self.forwarding
14302                .register_module_connection(
14303                    connection,
14304                    self.spec.module_id.clone(),
14305                    subc_protocol::PROTOCOL_VERSION,
14306                    Concurrency::ModuleManaged,
14307                    FrameSink::new(tx),
14308                )
14309                .unwrap();
14310            self.registry
14311                .register_with_control_ops(
14312                    manifest,
14313                    subc_protocol::PROTOCOL_VERSION,
14314                    connection,
14315                    if health {
14316                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14317                    } else {
14318                        vec![]
14319                    },
14320                )
14321                .unwrap();
14322        }
14323
14324        async fn probe(&mut self) -> crate::router::OutboundFrame {
14325            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14326                Ok(frame) => Some(frame),
14327                Err(mpsc::error::TryRecvError::Empty) => None,
14328                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14329            })
14330            .await;
14331            let body: Value = serde_json::from_slice(&frame.body).unwrap();
14332            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14333            frame
14334        }
14335
14336        fn no_probe(&mut self) {
14337            assert!(matches!(
14338                self.rx.try_recv(),
14339                Err(mpsc::error::TryRecvError::Empty)
14340            ));
14341        }
14342
14343        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14344            self.forwarding
14345                .complete_module_control_rpc(
14346                    ConnectionId::new(connection),
14347                    frame.header.corr,
14348                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14349                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14350                        status: HealthStatus::Ok,
14351                        detail: None,
14352                        metrics: None,
14353                    }),
14354                )
14355                .unwrap();
14356        }
14357    }
14358
14359    #[tokio::test(start_paused = true)]
14360    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14361        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14362        let initial = actor.module.status().unwrap();
14363        tokio::time::advance(Duration::from_secs(119)).await;
14364        tokio::task::yield_now().await;
14365        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14366        assert!(actor.module.terminal_history().entries.is_empty());
14367        actor.expire_registration_wait(Duration::from_secs(1)).await;
14368        actor
14369            .wait_for_select(|at| at.generation > initial.spawn_generation)
14370            .await;
14371        let status = actor.module.status().unwrap();
14372        assert_ne!(status.pid, initial.pid);
14373        assert!(!cortexkit_test_support::process_alive(
14374            initial.pid.unwrap() as i32
14375        ));
14376        assert_eq!(status.restart_count, 1);
14377        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14378        let entries = actor.module.terminal_history().entries;
14379        assert_eq!(entries.len(), 1);
14380        assert_eq!(
14381            entries[0].disposition_detail.as_deref(),
14382            Some("registration_timeout")
14383        );
14384        actor.module.stop().await.unwrap();
14385    }
14386
14387    #[tokio::test(start_paused = true)]
14388    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14389        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14390        let pid = actor.module.status().unwrap().pid;
14391        tokio::time::advance(Duration::from_secs(119)).await;
14392        actor.hello(1, false);
14393        let registered = actor
14394            .wait_for_select(|at| at.registered_connection.is_some())
14395            .await;
14396        assert_eq!(
14397            registered.wake_after, None,
14398            "HELLO must cancel the deadline"
14399        );
14400        tokio::time::advance(Duration::from_secs(300)).await;
14401        tokio::task::yield_now().await;
14402        assert_eq!(actor.module.status().unwrap().pid, pid);
14403        assert!(actor.module.terminal_history().entries.is_empty());
14404        // A disconnect after the first HELLO must not re-arm a startup timer.
14405        actor
14406            .registry
14407            .deregister_connection(ConnectionId::new(1))
14408            .unwrap();
14409        actor
14410            .wait_for_select(|at| at.registered_connection.is_none())
14411            .await;
14412        tokio::time::advance(Duration::from_secs(300)).await;
14413        tokio::task::yield_now().await;
14414        assert_eq!(actor.module.status().unwrap().pid, pid);
14415        assert!(actor.module.terminal_history().entries.is_empty());
14416        actor.module.stop().await.unwrap();
14417
14418        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14419        let pid = transient.module.status().unwrap().pid;
14420        tokio::time::advance(Duration::from_secs(119)).await;
14421        let turns = transient.turns();
14422        // Both writes happen before the supervisor can refresh the registry.
14423        // An accepted HELLO must remain evidence after its connection is gone.
14424        transient.hello(1, false);
14425        transient
14426            .registry
14427            .deregister_connection(ConnectionId::new(1))
14428            .unwrap();
14429        let parked = transient.wait_for_select(|at| at.turn > turns).await;
14430        assert_eq!(
14431            parked.wake_after, None,
14432            "transient HELLO must cancel the deadline"
14433        );
14434        tokio::time::advance(Duration::from_secs(300)).await;
14435        tokio::task::yield_now().await;
14436        assert_eq!(transient.module.status().unwrap().pid, pid);
14437        assert!(transient.module.terminal_history().entries.is_empty());
14438        transient.module.stop().await.unwrap();
14439    }
14440
14441    #[tokio::test(start_paused = true)]
14442    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14443        let actor = Actor::start_with_policy(
14444            ModuleProtocol::Subc,
14445            Duration::from_secs(30),
14446            Duration::from_secs(10),
14447            RestartPolicy::new(1, Duration::ZERO),
14448        )
14449        .await;
14450        let generation = actor.module.status().unwrap().spawn_generation;
14451        actor
14452            .expire_registration_wait(Duration::from_secs(10))
14453            .await;
14454        actor.wait_for_select(|at| at.generation > generation).await;
14455        actor
14456            .expire_registration_wait(Duration::from_secs(10))
14457            .await;
14458        let retry = wait_for("failed retry schedule", || {
14459            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14460            (state.state == ModuleState::Failed)
14461                .then_some(state.failed_retry.pending)
14462                .flatten()
14463        })
14464        .await;
14465        assert_eq!(retry.delay, Duration::from_secs(600));
14466        assert_eq!(retry.attempt, 1);
14467        let entries = actor.module.terminal_history().entries;
14468        assert_eq!(entries.len(), 2);
14469        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14470        assert!(entries[1]
14471            .disposition_detail
14472            .as_deref()
14473            .unwrap()
14474            .contains("registration_timeout"));
14475        tokio::time::advance(Duration::from_secs(599)).await;
14476        tokio::task::yield_now().await;
14477        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14478        tokio::time::advance(Duration::from_secs(1)).await;
14479        actor
14480            .wait_for_select(|at| at.generation > generation + 1)
14481            .await;
14482        // The slow retry is another ordinary spawn, with its own HELLO budget.
14483        actor
14484            .expire_registration_wait(Duration::from_secs(10))
14485            .await;
14486        wait_for("slow retry registration timeout", || {
14487            (actor.module.terminal_history().entries.len() == 3).then_some(())
14488        })
14489        .await;
14490        actor.module.stop().await.unwrap();
14491    }
14492
14493    #[tokio::test(start_paused = true)]
14494    async fn registration_timeout_honours_ten_second_override() {
14495        let actor = Actor::start_with_policy(
14496            ModuleProtocol::Subc,
14497            Duration::from_secs(30),
14498            Duration::from_secs(10),
14499            RestartPolicy::new(3, Duration::ZERO),
14500        )
14501        .await;
14502        let generation = actor.module.status().unwrap().spawn_generation;
14503        tokio::time::advance(Duration::from_secs(9)).await;
14504        tokio::task::yield_now().await;
14505        assert!(actor.module.terminal_history().entries.is_empty());
14506        actor.expire_registration_wait(Duration::from_secs(1)).await;
14507        actor.wait_for_select(|at| at.generation > generation).await;
14508        assert_eq!(actor.module.terminal_history().entries.len(), 1);
14509        actor.module.stop().await.unwrap();
14510    }
14511
14512    #[tokio::test]
14513    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14514        use std::os::unix::process::{CommandExt, ExitStatusExt};
14515
14516        struct Helper(std::process::Child);
14517        impl Drop for Helper {
14518            fn drop(&mut self) {
14519                // This handle still owns the helper until wait returns. Never
14520                // clean up its old process group after the leader is reaped.
14521                if matches!(self.0.try_wait(), Ok(None)) {
14522                    let _ = self.0.kill();
14523                    let _ = self.0.wait();
14524                }
14525            }
14526        }
14527
14528        let home = ScratchDir::new("registration-timeout-helper");
14529        let supervisor = Supervisor::new_for_test(
14530            Arc::new(Registry::default()),
14531            RestartPolicy::new(0, Duration::ZERO),
14532        );
14533        let spec = ModuleSpec {
14534            module_id: "registration-timeout-group".into(),
14535            program: "/bin/sleep".into(),
14536            args: vec!["120".into()],
14537            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14538                .into_iter()
14539                .map(|key| (key.into(), home.path().display().to_string()))
14540                .collect(),
14541            reserved: false,
14542            reserved_prefixes: vec![],
14543            protocol: ModuleProtocol::Subc,
14544            overlap: Default::default(),
14545        };
14546        let module = supervisor
14547            .supervise_configured_requested(
14548                spec,
14549                true,
14550                HealthConfig::default(),
14551                None,
14552                RestartPolicy::new(0, Duration::ZERO),
14553                Duration::from_secs(10),
14554                None,
14555            )
14556            .unwrap();
14557        let leader = timeout(Duration::from_secs(5), async {
14558            loop {
14559                if let Some(pid) = module.status().unwrap().pid {
14560                    break pid;
14561                }
14562                sleep(Duration::from_millis(5)).await;
14563            }
14564        })
14565        .await
14566        .expect("supervised group leader must be reported before the deadline");
14567        assert!(leader > 1);
14568        let ready = home.join("helper.ready");
14569        // Own the helper separately so even a broken group kill can be cleaned
14570        // up by its own unreaped child handle, without signalling a stale pgid.
14571        let mut helper = Helper(
14572            std::process::Command::new("/bin/sh")
14573                .args([
14574                    "-c",
14575                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14576                    "helper",
14577                ])
14578                .arg(&ready)
14579                .process_group(i32::try_from(leader).unwrap())
14580                .stdout(std::process::Stdio::null())
14581                .stderr(std::process::Stdio::null())
14582                .spawn()
14583                .unwrap(),
14584        );
14585        timeout(Duration::from_secs(5), async {
14586            while !ready.exists() {
14587                assert!(helper.0.try_wait().unwrap().is_none());
14588                sleep(Duration::from_millis(5)).await;
14589            }
14590        })
14591        .await
14592        .expect("helper must install TERM ignore before the registration deadline");
14593        let helper_pid =
14594            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14595        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14596        sleep(Duration::from_millis(20)).await;
14597        assert!(
14598            helper.0.try_wait().unwrap().is_none(),
14599            "helper must ignore TERM"
14600        );
14601
14602        timeout(Duration::from_secs(20), async {
14603            while module.state().unwrap() != ModuleState::Failed {
14604                sleep(Duration::from_millis(5)).await;
14605            }
14606        })
14607        .await
14608        .expect("registration timeout must stop and reap the group leader");
14609        let helper_exit = timeout(Duration::from_secs(3), async {
14610            loop {
14611                if let Some(status) = helper.0.try_wait().unwrap() {
14612                    break status;
14613                }
14614                sleep(Duration::from_millis(5)).await;
14615            }
14616        })
14617        .await
14618        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14619        assert_eq!(helper_exit.signal(), Some(9));
14620        assert!(!cortexkit_test_support::process_alive(
14621            i32::try_from(leader).unwrap()
14622        ));
14623        assert_eq!(
14624            module.status().unwrap().last_exit.unwrap().kind,
14625            ExitKind::Crash
14626        );
14627        assert!(module.terminal_history().entries[0]
14628            .disposition_detail
14629            .as_deref()
14630            .unwrap()
14631            .contains("registration_timeout"));
14632        module.stop().await.unwrap();
14633    }
14634
14635    #[tokio::test(start_paused = true)]
14636    async fn registration_timeout_does_not_count_an_earlier_exit() {
14637        let actor = Actor::start_with_policy(
14638            ModuleProtocol::Subc,
14639            Duration::from_secs(30),
14640            DEFAULT_REGISTER_TIMEOUT,
14641            RestartPolicy::new(0, Duration::ZERO),
14642        )
14643        .await;
14644        let pid = actor.module.status().unwrap().pid.unwrap();
14645        rustix::process::kill_process(
14646            rustix::process::Pid::from_raw(pid as i32).unwrap(),
14647            rustix::process::Signal::TERM,
14648        )
14649        .unwrap();
14650        wait_for("early exit", || {
14651            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14652        })
14653        .await;
14654        tokio::time::advance(Duration::from_secs(121)).await;
14655        tokio::task::yield_now().await;
14656        let entries = actor.module.terminal_history().entries;
14657        assert_eq!(entries.len(), 1);
14658        assert!(!entries[0]
14659            .disposition_detail
14660            .as_deref()
14661            .unwrap()
14662            .contains("registration_timeout"));
14663        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14664        actor.module.stop().await.unwrap();
14665    }
14666
14667    #[tokio::test(start_paused = true)]
14668    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14669        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14670        let pid = actor.module.status().unwrap().pid;
14671        tokio::time::advance(Duration::from_secs(119)).await;
14672        actor.roster.close();
14673        let turns = actor.turns();
14674        tokio::time::advance(Duration::from_secs(300)).await;
14675        let parked = actor.wait_for_select(|at| at.turn > turns).await;
14676        assert_eq!(parked.wake_after, None);
14677        assert_eq!(actor.module.status().unwrap().pid, pid);
14678        assert!(actor.module.terminal_history().entries.is_empty());
14679        actor.module.stop().await.unwrap();
14680        let status = actor.module.status().unwrap();
14681        assert_eq!(status.restart_count, 0);
14682        assert_eq!(
14683            actor.module.terminal_history().entries[0].disposition,
14684            TerminalDisposition::DaemonShutdown
14685        );
14686    }
14687
14688    #[tokio::test(start_paused = true)]
14689    async fn no_health_children_park_for_sixty_seconds() {
14690        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14691        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14692        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14693        unadvertised.hello(1, false);
14694        // Include any one-off registration wake in the budget; receiving a
14695        // no-health HELLO must not turn a parked child into a polling child.
14696        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
14697        // a single 60 s jump would only observe one expired timer.
14698        for _ in 0..6000 {
14699            tokio::time::advance(Duration::from_millis(10)).await;
14700            tokio::task::yield_now().await;
14701        }
14702        for actor in [&none, &unregistered, &unadvertised] {
14703            assert!(
14704                actor.turns() <= 5,
14705                "no-health actor woke {} times",
14706                actor.turns()
14707            );
14708            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14709        }
14710    }
14711
14712    #[tokio::test(start_paused = true)]
14713    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14714        // With a zero cadence the first probe is due at once, so this measures
14715        // only how fast the registration notification starts health probing,
14716        // not the normal 30-33 s wait before a first probe (checked below).
14717        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14718        let tick = Instant::now();
14719        actor.hello(1, true);
14720        actor.probe().await;
14721        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14722    }
14723
14724    #[tokio::test(start_paused = true)]
14725    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14726        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14727        let hello_tick = Instant::now();
14728        actor.hello(1, true);
14729        let armed = actor
14730            .wait_for_select(|checkpoint| {
14731                checkpoint.registered_connection == Some(ConnectionId::new(1))
14732                    && checkpoint.next_probe_at.is_some()
14733            })
14734            .await;
14735        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14736        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14737        let first_deadline = hello_tick + delay;
14738        assert_eq!(armed.next_probe_at, Some(first_deadline));
14739        tokio::time::advance(Duration::from_secs(29)).await;
14740        let turns = actor.turns();
14741        actor
14742            .registry
14743            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14744            .unwrap();
14745        let updated = actor
14746            .wait_for_select(|checkpoint| checkpoint.turn > turns)
14747            .await;
14748        assert_eq!(
14749            updated.next_probe_at,
14750            Some(first_deadline),
14751            "catalog update must not reset cadence"
14752        );
14753        actor.no_probe();
14754        tokio::time::advance(first_deadline - Instant::now()).await;
14755        let frame = actor.probe().await;
14756        actor.answer(1, frame);
14757        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14758        let next_deadline = Instant::now() + next;
14759        let rearmed = actor
14760            .wait_for_select(|checkpoint| {
14761                checkpoint
14762                    .next_probe_at
14763                    .is_some_and(|deadline| deadline > first_deadline)
14764            })
14765            .await;
14766        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14767        tokio::time::advance(next - Duration::from_nanos(1)).await;
14768        actor.no_probe();
14769        tokio::time::advance(Duration::from_nanos(1)).await;
14770        actor.probe().await;
14771    }
14772
14773    #[tokio::test(start_paused = true)]
14774    async fn disconnect_disarms_and_reconnect_rearms_health() {
14775        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14776        actor.hello(1, true);
14777        actor
14778            .wait_for_select(|checkpoint| {
14779                checkpoint.registered_connection == Some(ConnectionId::new(1))
14780                    && checkpoint.next_probe_at.is_some()
14781            })
14782            .await;
14783        let turns = actor.turns();
14784        let tick = Instant::now();
14785        actor
14786            .registry
14787            .deregister_connection(ConnectionId::new(1))
14788            .unwrap();
14789        let disarmed = actor
14790            .wait_for_select(|checkpoint| {
14791                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14792            })
14793            .await;
14794        assert_eq!(disarmed.next_probe_at, None);
14795        assert_eq!(disarmed.wake_after, None);
14796        assert_eq!(
14797            Instant::now(),
14798            tick,
14799            "disconnect must disarm in the notification tick"
14800        );
14801        tokio::time::advance(Duration::from_secs(60)).await;
14802        actor.no_probe();
14803        let reconnect_tick = Instant::now();
14804        actor.hello(2, true);
14805        let rearmed = actor
14806            .wait_for_select(|checkpoint| {
14807                checkpoint.registered_connection == Some(ConnectionId::new(2))
14808                    && checkpoint.next_probe_at.is_some()
14809            })
14810            .await;
14811        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14812        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14813        tokio::time::advance(delay).await;
14814        actor.probe().await;
14815    }
14816
14817    #[tokio::test(start_paused = true)]
14818    async fn reregistration_without_health_disarms_the_old_schedule() {
14819        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14820        actor.hello(1, true);
14821        actor
14822            .wait_for_select(|checkpoint| {
14823                checkpoint.registered_connection == Some(ConnectionId::new(1))
14824                    && checkpoint.next_probe_at.is_some()
14825            })
14826            .await;
14827        actor
14828            .registry
14829            .deregister_connection(ConnectionId::new(1))
14830            .unwrap();
14831        actor.hello(2, false);
14832        let disarmed = actor
14833            .wait_for_select(|checkpoint| {
14834                checkpoint.registered_connection == Some(ConnectionId::new(2))
14835            })
14836            .await;
14837        assert_eq!(disarmed.next_probe_at, None);
14838        assert_eq!(disarmed.wake_after, None);
14839        tokio::time::advance(Duration::from_secs(60)).await;
14840        actor.no_probe();
14841    }
14842
14843    #[tokio::test(start_paused = true)]
14844    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14845        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14846        actor.hello(1, true);
14847        let armed = actor
14848            .wait_for_select(|checkpoint| {
14849                checkpoint.registered_connection == Some(ConnectionId::new(1))
14850                    && checkpoint.next_probe_at.is_some()
14851            })
14852            .await;
14853        // Simulate the old process's connection closing before the restart
14854        // tears the process down. The replacement process has started but has
14855        // not sent its HELLO, so it is not registered.
14856        actor
14857            .registry
14858            .deregister_connection(ConnectionId::new(1))
14859            .unwrap();
14860        actor.module.restart(Some(0)).await.unwrap();
14861        let parked = actor
14862            .wait_for_select(|checkpoint| {
14863                checkpoint.generation > armed.generation
14864                    && checkpoint.registered_connection.is_none()
14865                    && checkpoint.next_probe_at.is_none()
14866            })
14867            .await;
14868        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
14869        actor.rx.close();
14870        // Drain the stop notice the restart queued for the old process's connection,
14871        // so it isn't mistaken for traffic to the replacement.
14872        while actor.rx.try_recv().is_ok() {}
14873        let turns = actor.turns();
14874        tokio::time::advance(Duration::from_secs(60)).await;
14875        let after = actor.turns();
14876        eprintln!(
14877            "restart park turns: before={turns} after={after} delta={}",
14878            after - turns
14879        );
14880        assert_eq!(after, turns, "replacement must park before HELLO");
14881        let hello_tick = Instant::now();
14882        actor.hello(2, true);
14883        let rearmed = actor
14884            .wait_for_select(|checkpoint| {
14885                checkpoint.registered_connection == Some(ConnectionId::new(2))
14886                    && checkpoint.next_probe_at.is_some()
14887            })
14888            .await;
14889        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14890        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14891        tokio::time::advance(delay).await;
14892        actor.probe().await;
14893    }
14894
14895    #[tokio::test(start_paused = true)]
14896    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14897        use tokio::{
14898            io::{AsyncReadExt, AsyncWriteExt},
14899            net::TcpListener,
14900        };
14901        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14902        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14903        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14904        let (started_tx, mut started_rx) = mpsc::channel(8);
14905        let peer = tokio::spawn(async move {
14906            loop {
14907                let (mut socket, _) = listener.accept().await.unwrap();
14908                let mut request = [0; 1024];
14909                assert!(socket.read(&mut request).await.unwrap() > 0);
14910                started_tx.send(Instant::now()).await.unwrap();
14911                socket
14912                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14913                    .await
14914                    .unwrap();
14915            }
14916        });
14917        actor
14918            .module
14919            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14920            .await
14921            .unwrap();
14922        let first_tick = Instant::now();
14923        let first = actor
14924            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14925            .await;
14926        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14927        assert!(started_rx.try_recv().is_err());
14928
14929        // Restart just before the old deadline: reusing it would give the new
14930        // process only one second to start, rather than a full cadence.
14931        tokio::time::advance(
14932            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14933        )
14934        .await;
14935        actor.module.restart(Some(0)).await.unwrap();
14936        let spawn_tick = Instant::now();
14937        let replacement = actor
14938            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14939            .await;
14940        let deadline = replacement.next_probe_at.unwrap();
14941        assert!(
14942            deadline >= spawn_tick + actor.health.cadence,
14943            "a replacement must not inherit its predecessor's probe deadline"
14944        );
14945        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14946        assert!(
14947            started_rx.try_recv().is_err(),
14948            "no HTTP probe during startup grace"
14949        );
14950        tokio::time::advance(deadline - Instant::now()).await;
14951        let landed = wait_for("first replacement HTTP probe", || {
14952            started_rx.try_recv().ok()
14953        })
14954        .await;
14955        assert!(landed >= spawn_tick + actor.health.cadence);
14956        wait_for("healthy replacement", || {
14957            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14958                .then_some(())
14959        })
14960        .await;
14961        peer.abort();
14962    }
14963
14964    #[tokio::test(start_paused = true)]
14965    async fn rescan_adds_and_removes_http_health_without_polling() {
14966        use tokio::{
14967            io::{AsyncReadExt, AsyncWriteExt},
14968            net::TcpListener,
14969        };
14970        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14971        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14972        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14973        // One millisecond separates successive probes without a polling delay.
14974        actor.health.cadence = Duration::from_millis(1);
14975        let (started_tx, mut started_rx) = mpsc::channel(8);
14976        let peer = tokio::spawn(async move {
14977            loop {
14978                let (mut socket, _) = listener.accept().await.unwrap();
14979                let mut request = [0; 1024];
14980                assert!(socket.read(&mut request).await.unwrap() > 0);
14981                started_tx.send(()).await.unwrap();
14982                socket
14983                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14984                    .await
14985                    .unwrap();
14986            }
14987        });
14988        let tick = Instant::now();
14989        let turns = actor.turns();
14990        actor
14991            .module
14992            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14993            .await
14994            .unwrap();
14995        let armed = actor
14996            .wait_for_select(|checkpoint| {
14997                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14998            })
14999            .await;
15000        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15001        assert_eq!(Instant::now(), tick);
15002        tokio::time::advance(Duration::from_millis(1)).await;
15003        let mut started = false;
15004        wait_for("successful HTTP probe", || {
15005            started |= started_rx.try_recv().is_ok();
15006            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15007                .then_some(())
15008        })
15009        .await;
15010        assert!(started, "added HTTP check must start at its first deadline");
15011        assert_eq!(
15012            actor.module.status().unwrap().health.status,
15013            SupervisorHealthStatus::Ok
15014        );
15015        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15016        actor.health.http = None;
15017        let turns = actor.turns();
15018        actor
15019            .module
15020            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15021            .await
15022            .unwrap();
15023        let parked = actor
15024            .wait_for_select(|checkpoint| {
15025                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15026            })
15027            .await;
15028        assert_eq!(parked.wake_after, None);
15029        let turns = actor.turns();
15030        tokio::time::advance(Duration::from_secs(60)).await;
15031        assert_eq!(
15032            actor.turns(),
15033            turns,
15034            "removed HTTP check must park the actor"
15035        );
15036        assert!(
15037            started_rx.try_recv().is_err(),
15038            "removed HTTP check must stay disarmed"
15039        );
15040        assert_eq!(
15041            actor.module.status().unwrap().health,
15042            ModuleHealthStatus::default()
15043        );
15044        peer.abort();
15045    }
15046}
15047
15048#[cfg(test)]
15049mod drain_busy_reprobe_tests {
15050    //! The drain's busy-gauge wait, driven against an in-process module
15051    //! connection whose health answers the test scripts. Time is paused, so
15052    //! every elapsed duration below is exact virtual time.
15053    use super::*;
15054    use std::collections::BTreeMap;
15055    use std::sync::Mutex as StdMutex;
15056    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15057
15058    const MODULE_ID: &str = "drain-reprobe-module";
15059    const CONNECTION: u64 = 7;
15060    const GAUGE: &str = "runs_in_flight";
15061    /// The production default for both the health cadence and the drain
15062    /// ceiling, which is the pairing that used to cost a full ceiling.
15063    const CADENCE: Duration = Duration::from_secs(30);
15064    const CEILING: Duration = Duration::from_secs(30);
15065
15066    /// How the scripted module answers each `health.check`.
15067    #[derive(Clone, Copy)]
15068    enum Answer {
15069        /// Report the gauge busy until this instant, quiet from then on, after
15070        /// taking `latency` to answer.
15071        BusyUntil { until: Instant, latency: Duration },
15072        /// Answer at once, with metrics that leave the declared gauge out.
15073        OmitsGauge,
15074        /// Never answer.
15075        Silent,
15076    }
15077
15078    struct Drain {
15079        forwarding: Arc<ForwardingTable>,
15080        runtime: SupervisorRuntimeConfig,
15081        endpoint: crate::ModuleEndpointId,
15082        /// When each probe reached the module.
15083        probes: Arc<StdMutex<Vec<Instant>>>,
15084        responder: tokio::task::JoinHandle<()>,
15085    }
15086
15087    impl Drain {
15088        fn start(answer: Answer) -> Self {
15089            let registry = Arc::new(Registry::default());
15090            let forwarding = Arc::new(ForwardingTable::default());
15091            let health = HealthConfig {
15092                cadence: CADENCE,
15093                ..HealthConfig::default()
15094            };
15095            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15096                .with_forwarding(forwarding.clone())
15097                .with_health_config(health)
15098                .runtime_config();
15099            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15100            let endpoint = forwarding
15101                .register_module_connection(
15102                    ConnectionId::new(CONNECTION),
15103                    MODULE_ID.to_string(),
15104                    subc_protocol::PROTOCOL_VERSION,
15105                    Concurrency::ModuleManaged,
15106                    FrameSink::new(tx),
15107                )
15108                .unwrap();
15109            let probes = Arc::new(StdMutex::new(Vec::new()));
15110            let responder = tokio::spawn({
15111                let forwarding = forwarding.clone();
15112                let probes = probes.clone();
15113                async move {
15114                    while let Some(frame) = rx.recv().await {
15115                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15116                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15117                        probes.lock().unwrap().push(Instant::now());
15118                        let metrics = match answer {
15119                            Answer::BusyUntil { until, latency } => {
15120                                sleep(latency).await;
15121                                let busy = u64::from(Instant::now() < until);
15122                                serde_json::json!({ GAUGE: busy })
15123                            }
15124                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15125                            Answer::Silent => continue,
15126                        };
15127                        // A probe abandoned at its deadline has no RPC left to
15128                        // complete, so a late answer may be refused.
15129                        let _ = forwarding.complete_module_control_rpc(
15130                            ConnectionId::new(CONNECTION),
15131                            frame.header.corr,
15132                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15133                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15134                                status: HealthStatus::Ok,
15135                                detail: None,
15136                                metrics: Some(metrics),
15137                            }),
15138                        );
15139                    }
15140                }
15141            });
15142            Self {
15143                forwarding,
15144                runtime,
15145                endpoint,
15146                probes,
15147                responder,
15148            }
15149        }
15150
15151        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15152        /// how long it took.
15153        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15154            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15155            (drained, elapsed)
15156        }
15157
15158        /// [`Self::run`], also returning what the drain's probes read.
15159        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15160            let started = Instant::now();
15161            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15162            let drained = wait_for_forwarding_quiescence(
15163                &self.forwarding,
15164                MODULE_ID,
15165                &self.runtime,
15166                self.endpoint,
15167                started + ceiling,
15168                &[GAUGE.to_string()],
15169                DrainScope::Active,
15170                &mut trace,
15171            )
15172            .await
15173            .unwrap();
15174            (drained, started.elapsed(), trace)
15175        }
15176
15177        fn probes(&self) -> Vec<Instant> {
15178            self.probes.lock().unwrap().clone()
15179        }
15180    }
15181
15182    impl Drop for Drain {
15183        fn drop(&mut self) {
15184            self.responder.abort();
15185        }
15186    }
15187
15188    #[tokio::test(start_paused = true)]
15189    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15190        let drain = Drain::start(Answer::BusyUntil {
15191            until: Instant::now() + Duration::from_secs(1),
15192            latency: Duration::ZERO,
15193        });
15194        let (drained, elapsed) = drain.run(CEILING).await;
15195        assert!(
15196            drained,
15197            "gauges that read zero during the drain must let it finish drained, \
15198             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15199        );
15200        assert!(
15201            elapsed < Duration::from_secs(3),
15202            "drain took {elapsed:?} after the gauges went quiet at 1s"
15203        );
15204        let probes = drain.probes();
15205        assert!(
15206            probes.len() >= 2,
15207            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15208        );
15209    }
15210
15211    /// The same shape in real time on a multi-threaded runtime: the first
15212    /// probe is answered busy at once, within a millisecond of the drain
15213    /// starting, and later probes read idle. The drain must keep probing
15214    /// after that first instant answer and end soon after the gauge falls.
15215    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15216    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15217        let drain = Drain::start(Answer::BusyUntil {
15218            until: Instant::now() + Duration::from_millis(600),
15219            latency: Duration::ZERO,
15220        });
15221        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15222        assert!(drained, "the gauge fell at 600ms: {trace}");
15223        assert!(
15224            elapsed < Duration::from_secs(3),
15225            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15226        );
15227        let probes = drain.probes();
15228        assert!(
15229            probes.len() >= 2 && trace.probes as usize == probes.len(),
15230            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15231        );
15232    }
15233
15234    #[tokio::test(start_paused = true)]
15235    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15236        let started = Instant::now();
15237        let drain = Drain::start(Answer::BusyUntil {
15238            until: started + Duration::from_secs(3600),
15239            latency: Duration::ZERO,
15240        });
15241        let (drained, elapsed) = drain.run(CEILING).await;
15242        assert!(!drained, "a module busy throughout must not drain");
15243        assert!(
15244            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15245            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15246        );
15247        // Re-probing is paced by the interval, not a busy loop: at most one
15248        // probe per interval across the ceiling, plus the first at time zero.
15249        let probes = drain.probes();
15250        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15251        assert!(
15252            probes.len() as u128 <= max_probes,
15253            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15254            probes.len()
15255        );
15256        for pair in probes.windows(2) {
15257            assert!(
15258                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15259                "consecutive probes {:?} apart",
15260                pair[1] - pair[0]
15261            );
15262        }
15263    }
15264
15265    #[tokio::test(start_paused = true)]
15266    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15267        // Each answer takes longer than the re-probe interval; the next probe
15268        // must wait for the previous answer and then the interval.
15269        let latency = Duration::from_secs(1);
15270        let drain = Drain::start(Answer::BusyUntil {
15271            until: Instant::now() + Duration::from_secs(3600),
15272            latency,
15273        });
15274        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15275        assert!(!drained);
15276        let probes = drain.probes();
15277        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15278        for pair in probes.windows(2) {
15279            assert!(
15280                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15281                "a probe started {:?} after the previous one, before it could finish",
15282                pair[1] - pair[0]
15283            );
15284        }
15285    }
15286
15287    #[tokio::test(start_paused = true)]
15288    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15289        let drain = Drain::start(Answer::Silent);
15290        let ceiling = Duration::from_secs(2);
15291        let (drained, elapsed) = drain.run(ceiling).await;
15292        assert!(!drained, "an unanswered probe counts as busy");
15293        assert!(
15294            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15295            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15296        );
15297        assert_eq!(
15298            drain.probes().len(),
15299            1,
15300            "the unanswered probe holds the only slot until the ceiling"
15301        );
15302    }
15303
15304    /// Every event logged while it is installed, as `message` plus fields.
15305    #[derive(Clone, Default)]
15306    struct LogCapture {
15307        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15308    }
15309
15310    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15311        fn on_event(
15312            &self,
15313            event: &tracing::Event<'_>,
15314            _context: tracing_subscriber::layer::Context<'_, S>,
15315        ) {
15316            struct Fields(BTreeMap<String, String>);
15317            impl tracing::field::Visit for Fields {
15318                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15319                    self.0
15320                        .insert(field.name().to_string(), format!("{value:?}"));
15321                }
15322            }
15323            let mut fields = Fields(BTreeMap::new());
15324            event.record(&mut fields);
15325            self.events.lock().unwrap().push(fields.0);
15326        }
15327    }
15328
15329    impl LogCapture {
15330        fn install(&self) -> tracing::subscriber::DefaultGuard {
15331            use tracing_subscriber::layer::SubscriberExt;
15332            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15333        }
15334
15335        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15336            self.events
15337                .lock()
15338                .unwrap()
15339                .iter()
15340                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15341                .cloned()
15342                .collect()
15343        }
15344    }
15345
15346    /// A module that declares a busy gauge but leaves it out of its health
15347    /// answers holds a drain to its ceiling. The timeout line must say so by
15348    /// name, and the drain must warn about the missing gauge once, not on
15349    /// every probe.
15350    #[tokio::test(start_paused = true)]
15351    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15352        let capture = LogCapture::default();
15353        let _guard = capture.install();
15354        let drain = Drain::start(Answer::OmitsGauge);
15355        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15356        assert!(!drained, "a gauge never reported cannot read quiet");
15357        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15358
15359        let missing = capture.with_message("lacks a declared busy gauge");
15360        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15361        assert_eq!(
15362            missing[0].get("missing_gauges").map(String::as_str),
15363            Some(GAUGE)
15364        );
15365
15366        warn_drain_timed_out(
15367            MODULE_ID,
15368            Duration::from_secs(2),
15369            RouteCloseReason::Restart,
15370            &Default::default(),
15371            &trace,
15372        );
15373        let timeouts = capture.with_message("route drain timed out");
15374        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15375        let gauges = &timeouts[0]["busy_gauges"];
15376        assert!(
15377            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15378            "the timeout line must name the gauge and why it read busy: {gauges}"
15379        );
15380        assert!(
15381            gauges.ends_with("; no value ever read"),
15382            "no value was ever read for the gauge: {gauges}"
15383        );
15384        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15385    }
15386
15387    /// The trace names a busy value and when it was read. For a probe that
15388    /// never got an answer, it names the probe's error kind and says that no
15389    /// value was ever read.
15390    #[tokio::test(start_paused = true)]
15391    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15392        let busy = Drain::start(Answer::BusyUntil {
15393            until: Instant::now() + Duration::from_secs(3600),
15394            latency: Duration::ZERO,
15395        });
15396        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15397        let rendered = trace.to_string();
15398        assert!(
15399            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15400                && rendered.contains("; last value non_zero(1) @"),
15401            "{rendered}"
15402        );
15403
15404        let silent = Drain::start(Answer::Silent);
15405        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15406        // The probe is capped at the 1 s ceiling, and its reading is stamped
15407        // when it gave up.
15408        assert_eq!(
15409            trace.to_string(),
15410            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15411        );
15412        assert_eq!(trace.probes, 1);
15413    }
15414
15415    /// A gauge value that is not a count reads busy and is shown as sent.
15416    #[test]
15417    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15418        let gauges = [GAUGE.to_string()];
15419        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15420        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15421        assert_eq!(
15422            busy_gauge_observation(&verdicts),
15423            BusyGaugeObservation::Busy
15424        );
15425        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15426        assert_eq!(
15427            busy_gauge_observation(&zero),
15428            BusyGaugeObservation::Quiescent
15429        );
15430        let none = gauge_verdicts(None, &gauges);
15431        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15432    }
15433}
15434
15435#[cfg(test)]
15436mod health_evidence_tests {
15437    use super::{HealthProbeError, HealthProbeEvidence};
15438    use std::collections::HashSet;
15439
15440    /// The evidential asymmetry, asserted rather than described.
15441    ///
15442    /// Exactly ONE observation is proof a module cannot serve, and the one that
15443    /// fires under CPU starvation is not it. Before the split, all fifteen
15444    /// construction sites collapsed into a single String, so a timeout carried the
15445    /// same weight as a dead lane -- which is how a healthy module was restarted
15446    /// three times in one day.
15447    #[test]
15448    fn only_a_dead_lane_is_proof_of_death() {
15449        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15450        // Three non-proof classes, each for a different reason: silence is
15451        // consistent with health, a bad answer proves the module ALIVE, and a
15452        // daemon-side fault never reached the module at all.
15453        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15454        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15455        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15456    }
15457
15458    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
15459    ///
15460    /// A shared label renders two different observations identically in the line an
15461    /// operator reads after an unexplained restart -- the exact confusion this
15462    /// change removes.
15463    #[test]
15464    fn every_evidence_class_has_a_distinct_label() {
15465        let labels = [
15466            HealthProbeError::lane_dead("").label(),
15467            HealthProbeError::no_answer("").label(),
15468            HealthProbeError::bad_answer("").label(),
15469            HealthProbeError::misconfigured("").label(),
15470        ];
15471        let unique: HashSet<_> = labels.iter().collect();
15472        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15473    }
15474
15475    /// The class is additional information, not a replacement.
15476    ///
15477    /// An operator needs both "this was silence" and the specific text saying how
15478    /// long we waited; a classification that swallowed the message would trade one
15479    /// missing distinction for another.
15480    #[test]
15481    fn classification_preserves_the_original_message() {
15482        let err = HealthProbeError::no_answer("module did not answer within 5s");
15483        assert_eq!(err.to_string(), "module did not answer within 5s");
15484        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15485    }
15486}
15487
15488#[cfg(test)]
15489mod health_tombstone_tests {
15490    use std::{path::PathBuf, sync::Arc, time::Duration};
15491
15492    use subc_protocol::{
15493        manifest::Concurrency,
15494        session::{HealthStatus, ModuleControlResponse},
15495    };
15496    use tokio::sync::mpsc;
15497
15498    use super::{
15499        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15500        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15501    };
15502    use crate::{
15503        control::ControlHandler,
15504        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15505        registry::{ConnectionId, Registry},
15506        router::FrameSink,
15507    };
15508
15509    struct ProbeHarness {
15510        spec: ModuleSpec,
15511        runtime: SupervisorRuntimeConfig,
15512        forwarding: Arc<ForwardingTable>,
15513        module_connection: ConnectionId,
15514        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15515        handler: ControlHandler,
15516        module: super::SupervisedModule,
15517    }
15518
15519    fn probe_harness() -> ProbeHarness {
15520        let registry = Arc::new(Registry::default());
15521        let forwarding = Arc::new(ForwardingTable::default());
15522        let supervisor_handle = super::SupervisorHandle::new();
15523        let health = HealthConfig {
15524            http: None,
15525            cadence: Duration::from_secs(30),
15526            deadline: Duration::from_secs(5),
15527            failure_threshold: 3,
15528            on_degraded: HealthAction::Report,
15529            on_failing: HealthAction::Report,
15530            critical: false,
15531        };
15532        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15533            .with_forwarding(Arc::clone(&forwarding))
15534            .with_handle(supervisor_handle.clone())
15535            .with_health_config(health);
15536        let spec = ModuleSpec {
15537            module_id: "late-health-module".to_string(),
15538            program: PathBuf::from("disabled-module"),
15539            args: Vec::new(),
15540            env: Vec::new(),
15541            reserved: false,
15542            reserved_prefixes: Vec::new(),
15543            protocol: ModuleProtocol::Subc,
15544            overlap: Default::default(),
15545        };
15546        let module = supervisor
15547            .supervise_configured(spec.clone(), false)
15548            .unwrap();
15549        let runtime = supervisor.runtime_config();
15550        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15551            .with_supervisor(supervisor_handle);
15552        let module_connection = ConnectionId::new(700);
15553        let (module_tx, module_rx) = mpsc::channel(8);
15554        forwarding
15555            .register_module_connection(
15556                module_connection,
15557                spec.module_id.clone(),
15558                subc_protocol::PROTOCOL_VERSION,
15559                Concurrency::ModuleManaged,
15560                FrameSink::new(module_tx),
15561            )
15562            .unwrap();
15563
15564        ProbeHarness {
15565            spec,
15566            runtime,
15567            forwarding,
15568            module_connection,
15569            module_rx,
15570            handler,
15571            module,
15572        }
15573    }
15574
15575    async fn finish_after(
15576        harness: &mut ProbeHarness,
15577        stall: Duration,
15578    ) -> ModuleControlRpcCompletion {
15579        assert!(stall > harness.runtime.health.deadline);
15580        let deadline = harness.runtime.health.deadline;
15581        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15582        let answer = async {
15583            let frame = harness.module_rx.recv().await.expect("health.check frame");
15584            tokio::time::advance(deadline).await;
15585            tokio::task::yield_now().await;
15586            tokio::time::advance(stall - deadline).await;
15587            harness
15588                .forwarding
15589                .complete_module_control_rpc(
15590                    harness.module_connection,
15591                    frame.header.corr,
15592                    Some("health.check"),
15593                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15594                        status: HealthStatus::Ok,
15595                        detail: None,
15596                        metrics: None,
15597                    }),
15598                )
15599                .unwrap()
15600        };
15601        let (probe_result, completion) = tokio::join!(probe, answer);
15602        let err = probe_result.expect_err("probe must miss its deadline");
15603        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15604        completion
15605    }
15606
15607    async fn time_out_without_answer(harness: &mut ProbeHarness) {
15608        let deadline = harness.runtime.health.deadline;
15609        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15610        let exhaust_deadline = async {
15611            let _frame = harness.module_rx.recv().await.expect("health.check frame");
15612            tokio::time::advance(deadline).await;
15613            tokio::task::yield_now().await;
15614        };
15615        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15616        let err = probe_result.expect_err("probe must miss its deadline");
15617        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15618    }
15619
15620    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15621        let registry = Arc::clone(&harness.module.inner.registry);
15622        let snapshot = Arc::clone(&harness.module.inner.snapshot);
15623        let process_liveness = super::SupervisorProcessLiveness::default();
15624        let mut child = None;
15625        let cycle = super::run_health_probe_cycle(
15626            &harness.spec,
15627            &harness.runtime,
15628            &registry,
15629            &process_liveness,
15630            &snapshot,
15631            &mut child,
15632        );
15633        let peer = async {
15634            let frame = harness.module_rx.recv().await.expect("health.check frame");
15635            if answer {
15636                harness
15637                    .forwarding
15638                    .complete_module_control_rpc(
15639                        harness.module_connection,
15640                        frame.header.corr,
15641                        Some("health.check"),
15642                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15643                            status: HealthStatus::Ok,
15644                            detail: None,
15645                            metrics: Some(serde_json::json!({"ready": true})),
15646                        }),
15647                    )
15648                    .unwrap();
15649            } else {
15650                tokio::time::advance(harness.runtime.health.deadline).await;
15651                tokio::task::yield_now().await;
15652            }
15653        };
15654        tokio::join!(cycle, peer);
15655    }
15656
15657    #[tokio::test(start_paused = true)]
15658    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15659        let mut harness = probe_harness();
15660        // Drive the probe cycle directly with an in-memory wire peer. Stop the
15661        // disabled module's monitor so only this test owns lifecycle transitions;
15662        // no OS process is launched, and a restart is observed at scheduling.
15663        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15664        monitor.abort();
15665        let _ = monitor.await;
15666        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15667            state.enabled = true;
15668            state.state = super::ModuleState::Running;
15669            state.process_alive = true;
15670        })
15671        .unwrap();
15672
15673        run_probe_cycle(&mut harness, true).await;
15674        assert_eq!(
15675            harness.module.status().unwrap().health.status,
15676            super::SupervisorHealthStatus::Ok
15677        );
15678
15679        for failures in 1..harness.runtime.health.failure_threshold {
15680            run_probe_cycle(&mut harness, false).await;
15681            let status = harness.module.status().unwrap();
15682            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15683            assert_eq!(status.health.consecutive_failures, failures);
15684            assert!(status.health.last_probe_ms.is_some());
15685            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15686            assert!(status.health.metrics.is_none());
15687            assert_eq!(status.state, super::ModuleState::Running);
15688            assert!(status.process_alive);
15689            assert_eq!(status.restart_count, 0);
15690            assert_eq!(status.lifetime_restarts, 0);
15691            assert!(status.health.last_action.is_none());
15692            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15693        }
15694
15695        run_probe_cycle(&mut harness, true).await;
15696        let recovered = harness.module.status().unwrap();
15697        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15698        assert_eq!(recovered.health.consecutive_failures, 0);
15699        assert!(recovered.health.detail.is_none());
15700        assert_eq!(
15701            recovered.health.metrics,
15702            Some(serde_json::json!({"ready": true}))
15703        );
15704        assert_eq!(recovered.lifetime_restarts, 0);
15705
15706        for failures in 1..=harness.runtime.health.failure_threshold {
15707            run_probe_cycle(&mut harness, false).await;
15708            let status = harness.module.status().unwrap();
15709            assert_eq!(status.health.consecutive_failures, failures);
15710            if failures < harness.runtime.health.failure_threshold {
15711                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15712                assert_eq!(status.state, super::ModuleState::Running);
15713                assert_eq!(status.lifetime_restarts, 0);
15714                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15715            } else {
15716                assert_eq!(
15717                    status.health.status,
15718                    super::SupervisorHealthStatus::Unresponsive
15719                );
15720                assert_eq!(status.state, super::ModuleState::Restarting);
15721                assert_eq!(status.restart_count, 1);
15722                assert_eq!(status.lifetime_restarts, 1);
15723                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15724                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15725            }
15726        }
15727    }
15728
15729    #[tokio::test(start_paused = true)]
15730    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15731        let mut harness = probe_harness();
15732
15733        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15734        let first_latency = match &first {
15735            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15736            other => panic!("late answer was not retained: {other:?}"),
15737        };
15738        assert!(harness.handler.observe_module_control_completion(first));
15739
15740        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15741        let second_latency = match &second {
15742            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15743            other => panic!("late answer was not retained: {other:?}"),
15744        };
15745        assert!(harness.handler.observe_module_control_completion(second));
15746
15747        assert_eq!(first_latency, Duration::from_secs(8));
15748        assert_eq!(
15749            second_latency - first_latency,
15750            Duration::from_secs(3),
15751            "latency must grow linearly with the additional stall"
15752        );
15753        let health = harness.module.status().unwrap().health;
15754        assert_eq!(health.late_answer_count, 2);
15755        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15756    }
15757
15758    /// A module that answers every probe late must never march to the kill
15759    /// threshold: the late answer proves it is alive, so it must clear the miss
15760    /// streak the timeout recorded. Without the reset, a CPU-starved module
15761    /// that serves every probe seconds past the deadline accumulates
15762    /// `consecutive_failures` to the threshold and is killed — the exact
15763    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15764    /// "proves the module is alive" five times while counting five misses.
15765    #[tokio::test(start_paused = true)]
15766    async fn late_answer_clears_the_consecutive_failure_streak() {
15767        let mut harness = probe_harness();
15768
15769        // Timeout recorded first: the probe path saw no answer in time.
15770        time_out_without_answer(&mut harness).await;
15771        harness
15772            .module
15773            .record_health_probe_failure_for_test("[no-answer] test miss")
15774            .unwrap();
15775        assert_eq!(
15776            harness.module.status().unwrap().health.consecutive_failures,
15777            1,
15778            "precondition: the miss must be on the streak before the late answer"
15779        );
15780
15781        // The stalled reply then lands: proof of life.
15782        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15783        assert!(matches!(
15784            late,
15785            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15786        ));
15787        assert!(harness.handler.observe_module_control_completion(late));
15788
15789        let health = harness.module.status().unwrap().health;
15790        assert_eq!(
15791            health.consecutive_failures, 0,
15792            "a late answer is an answer: the streak must reset"
15793        );
15794        assert_eq!(health.late_answer_count, 1);
15795    }
15796
15797    #[tokio::test(start_paused = true)]
15798    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15799        let mut harness = probe_harness();
15800
15801        for _ in 0..20 {
15802            time_out_without_answer(&mut harness).await;
15803            assert_eq!(
15804                harness.forwarding.health_probe_tombstone_count().unwrap(),
15805                1
15806            );
15807        }
15808    }
15809}
15810
15811#[cfg(test)]
15812mod child_env_tests {
15813    #[cfg(windows)]
15814    use super::{
15815        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15816    };
15817    #[cfg(windows)]
15818    use std::sync::{Arc, OnceLock};
15819    #[cfg(windows)]
15820    #[test]
15821    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15822        let spec = spec(Vec::new());
15823        let mut command = Command::new("unused");
15824        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15825            .unwrap()
15826            .unwrap();
15827        let name = command
15828            .as_std()
15829            .get_envs()
15830            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15831            .and_then(|(_, value)| value)
15832            .unwrap();
15833        assert_eq!(name, OsStr::new(handoff.name()));
15834        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15835        assert!(command.as_std().get_envs().any(|(key, value)| key
15836            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15837            && value == Some(OsStr::new("env"))));
15838        assert!(command
15839            .as_std()
15840            .get_envs()
15841            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15842    }
15843
15844    #[cfg(windows)]
15845    #[test]
15846    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15847        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15848        snapshot.process_alive = true;
15849        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15850        assert_eq!(
15851            observed_launch_nonce_source(&snapshot).as_deref(),
15852            Some("env")
15853        );
15854        let consumed = Arc::new(OnceLock::new());
15855        snapshot.pipe_consumed = Some(consumed.clone());
15856        assert_eq!(
15857            observed_launch_nonce_source(&snapshot).as_deref(),
15858            Some("env")
15859        );
15860        consumed.set(()).unwrap();
15861        assert_eq!(
15862            observed_launch_nonce_source(&snapshot).as_deref(),
15863            Some("pipe")
15864        );
15865        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15866        assert_eq!(
15867            observed_launch_nonce_source(&snapshot).as_deref(),
15868            Some("env")
15869        );
15870        clear_current_process_facts(&mut snapshot);
15871        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15872    }
15873    use super::{
15874        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15875        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15876        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15877    };
15878    use std::{ffi::OsStr, path::PathBuf};
15879    use tokio::process::Command;
15880
15881    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15882        ModuleSpec {
15883            module_id: "env-plan".to_string(),
15884            program: PathBuf::from("/nonexistent"),
15885            args: Vec::new(),
15886            env,
15887            reserved: false,
15888            reserved_prefixes: Vec::new(),
15889            protocol: ModuleProtocol::Subc,
15890            overlap: Default::default(),
15891        }
15892    }
15893
15894    #[test]
15895    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15896        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15897        let digest = LaunchSpecDigest::from_spec(&base);
15898        let mut reordered = base.clone();
15899        reordered.env.reverse();
15900        reordered.module_id = "different-id".into();
15901        reordered.reserved = true;
15902        reordered.overlap = Default::default();
15903        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15904        let mut delimiter_collision = base.clone();
15905        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15906        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15907        let mut changed = base.clone();
15908        changed.protocol = ModuleProtocol::None;
15909        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15910        changed = base.clone();
15911        changed.args.push("--flag".into());
15912        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15913        changed = base.clone();
15914        changed.program = PathBuf::from("/other");
15915        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15916        changed = base.clone();
15917        changed.env[1].1 = "other".into();
15918        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15919        assert!(!format!("{digest:?}").contains("B=C"));
15920        assert!(!format!("{digest:?}").contains("[u8"));
15921    }
15922
15923    #[cfg(unix)]
15924    #[test]
15925    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15926        use std::os::unix::ffi::OsStringExt;
15927        let mut left = spec(Vec::new());
15928        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15929        let mut right = left.clone();
15930        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15931        assert_eq!(
15932            left.program.to_string_lossy(),
15933            right.program.to_string_lossy()
15934        );
15935        assert_ne!(
15936            LaunchSpecDigest::from_spec(&left),
15937            LaunchSpecDigest::from_spec(&right)
15938        );
15939    }
15940
15941    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
15942    /// one still gets its own.
15943    ///
15944    /// This is the narrow goal `env_clear()` was reached for, and the reason the
15945    /// fix is `env_remove` rather than deleting the line: an operator's ambient
15946    /// filter silently becoming an unconfigured module's log level is a real
15947    /// defect, just a much smaller one than clearing the environment.
15948    ///
15949    /// Asserted on the command plan rather than a spawned child because proving
15950    /// the ABSENCE of an inherited variable needs the parent's environment
15951    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
15952    /// removal as `(key, None)`, which is exactly the distinction wanted: not
15953    /// "absent because nobody set it" but "explicitly unset for the child".
15954    #[test]
15955    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15956        let mut command = Command::new("/nonexistent");
15957        apply_child_env(&mut command, &spec(Vec::new()));
15958        let removed = command
15959            .as_std()
15960            .get_envs()
15961            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15962        assert!(
15963            removed,
15964            "ambient CK_LOG must be explicitly removed for an unconfigured module"
15965        );
15966
15967        let mut configured = Command::new("/nonexistent");
15968        apply_child_env(
15969            &mut configured,
15970            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15971        );
15972        let effective = configured
15973            .as_std()
15974            .get_envs()
15975            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15976            .last()
15977            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15978        assert_eq!(
15979            effective,
15980            Some(Some("debug".to_string())),
15981            "a module's configured CK_LOG must survive the ambient removal"
15982        );
15983    }
15984
15985    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
15986    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
15987    /// the same reason as the CK_LOG test above.
15988    ///
15989    /// The argument is the load-bearing half: a stock binary exits on an
15990    /// unknown flag before it listens, so with `--subc` appended the mode
15991    /// could not supervise the one process it exists for. Found by the first
15992    /// conformance run (nats-server: `flag provided but not defined: -subc`).
15993    #[test]
15994    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15995        let connection_file = std::path::Path::new("/run/subc-connection.json");
15996        let handle = SupervisorHandle::new();
15997
15998        let mut none_spec = spec(Vec::new());
15999        none_spec.protocol = ModuleProtocol::None;
16000        let mut none = Command::new("/nonexistent");
16001        let none_handoff =
16002            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16003                .expect("protocol-none spawn args apply");
16004        assert!(
16005            none_handoff.is_none(),
16006            "protocol:none spawn must not receive a nonce descriptor"
16007        );
16008        #[cfg(windows)]
16009        assert!(!none.as_std().get_envs().any(|(key, value)| key
16010            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16011            && value.is_some()));
16012        assert!(
16013            !none.as_std().get_envs().any(|(key, value)| key
16014                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16015                && value.is_some()),
16016            "protocol:none spawn must not name a nonce descriptor"
16017        );
16018        let none_args: Vec<String> = none
16019            .as_std()
16020            .get_args()
16021            .map(|a| a.to_string_lossy().into_owned())
16022            .collect();
16023        assert!(
16024            !none_args.iter().any(|a| a == SUBC_ARG),
16025            "protocol:none argv must not carry --subc; got {none_args:?}"
16026        );
16027        let none_has_nonce = none
16028            .as_std()
16029            .get_envs()
16030            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16031        assert!(
16032            !none_has_nonce,
16033            "protocol:none spawn must not receive a launch nonce"
16034        );
16035        let none_has_module_id = none
16036            .as_std()
16037            .get_envs()
16038            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16039        assert!(
16040            none_has_module_id,
16041            "SUBC_MODULE_ID is inert and stays on every path"
16042        );
16043        assert!(
16044            handle.spawn_nonce(&none_spec.module_id).is_none(),
16045            "no nonce record for a process that will never present one"
16046        );
16047
16048        // Control: the subc-wire path is unchanged by the branch above.
16049        let wire_spec = spec(Vec::new());
16050        let mut wire = Command::new("/nonexistent");
16051        let wire_handoff =
16052            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16053                .expect("subc-wire spawn args apply");
16054        let wire_fd_env = wire
16055            .as_std()
16056            .get_envs()
16057            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16058            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16059        #[cfg(unix)]
16060        assert_eq!(
16061            wire_fd_env,
16062            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16063            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16064        );
16065        #[cfg(windows)]
16066        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16067        #[cfg(not(any(unix, windows)))]
16068        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16069        let wire_args: Vec<String> = wire
16070            .as_std()
16071            .get_args()
16072            .map(|a| a.to_string_lossy().into_owned())
16073            .collect();
16074        assert_eq!(
16075            wire_args,
16076            vec![
16077                SUBC_ARG.to_string(),
16078                connection_file.to_string_lossy().into_owned()
16079            ],
16080            "a subc-wire spawn still carries --subc <path>"
16081        );
16082        assert_eq!(
16083            wire.as_std()
16084                .get_envs()
16085                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16086            !cfg!(unix),
16087            "only Windows supplies the environment nonce"
16088        );
16089        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16090    }
16091
16092    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16093    /// spec tries to set it; only a swap candidate carries it.
16094    ///
16095    /// "Set it only on candidates" is not enough, because spawn applies the
16096    /// spec's env verbatim and the daemon's own environment is inherited: either
16097    /// could hand a plain restart the swap role, and a module reading it would
16098    /// warm on its long swap budget while callers wait. Asserted as an explicit
16099    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16100    /// test above gives.
16101    #[test]
16102    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16103        let role = |command: &Command| {
16104            command
16105                .as_std()
16106                .get_envs()
16107                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16108                .last()
16109                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16110        };
16111        let forged = spec(vec![(
16112            SUBC_SPAWN_ROLE_ENV.to_string(),
16113            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16114        )]);
16115
16116        let mut plain = Command::new("/nonexistent");
16117        apply_child_env(&mut plain, &forged);
16118        apply_spawn_role(&mut plain, SpawnRole::Plain);
16119        assert_eq!(
16120            role(&plain),
16121            Some(None),
16122            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16123        );
16124
16125        let mut candidate = Command::new("/nonexistent");
16126        apply_child_env(&mut candidate, &spec(Vec::new()));
16127        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16128        assert_eq!(
16129            role(&candidate),
16130            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16131        );
16132    }
16133
16134    /// Daemon-private capture retention keys never reach the child.
16135    ///
16136    /// cortexkit-log exposes retention as a Rust struct with no environment
16137    /// names, so these entries are supervisor metadata. Passing them through
16138    /// would invent a public child-process contract by accident.
16139    #[test]
16140    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16141        let mut command = Command::new("/nonexistent");
16142        apply_child_env(
16143            &mut command,
16144            &spec(vec![
16145                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16146                ("KEPT".to_string(), "yes".to_string()),
16147            ]),
16148        );
16149        let keys: Vec<String> = command
16150            .as_std()
16151            .get_envs()
16152            .filter(|(_, value)| value.is_some())
16153            .map(|(key, _)| key.to_string_lossy().into_owned())
16154            .collect();
16155        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16156        assert!(
16157            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16158            "daemon-private capture key leaked to the child: {keys:?}"
16159        );
16160    }
16161}
16162
16163#[cfg(test)]
16164mod jitter_tests {
16165    use super::jittered_health_delay;
16166    use std::{collections::HashSet, time::Duration};
16167
16168    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16169    /// that actually occur rather than invented ones.
16170    ///
16171    /// This is a SAMPLE, not a registry: the property under test is that distinct
16172    /// ids disperse, which holds for any set of distinct strings. Several entries
16173    /// are already historical (modules get renamed), and that costs nothing here --
16174    /// but it means a reader must not mistake this for the live module set, and a
16175    /// rename sweep will match it without there being anything to change.
16176    const FLEET: [&str; 14] = [
16177        "aft",
16178        "alfonso-core",
16179        "magic-context",
16180        "broca",
16181        "thalamus",
16182        "quota",
16183        "engram",
16184        "plexus",
16185        "cerebellum",
16186        "astrocyte",
16187        "synapse",
16188        "subc-mcp",
16189        "cortexkit-credentials",
16190        "subc-federation",
16191    ];
16192
16193    /// Probes must not converge after a fleet-wide restart.
16194    ///
16195    /// This is the property the jitter exists for: every module reconnects at
16196    /// once, and without dispersal all fourteen would then probe on the same
16197    /// tick forever. Nothing failed visibly when this went untested -- a
16198    /// convergent fleet still probes correctly, just in a burst, so the symptom
16199    /// is a periodic load spike that looks like whatever else is running.
16200    #[test]
16201    fn probe_delays_disperse_across_the_fleet() {
16202        let cadence = Duration::from_secs(30);
16203        let delays: HashSet<Duration> = FLEET
16204            .iter()
16205            .map(|id| jittered_health_delay(id, 0, cadence))
16206            .collect();
16207        assert_eq!(
16208            delays.len(),
16209            FLEET.len(),
16210            "every supervised module must land on its own probe offset"
16211        );
16212    }
16213
16214    /// The offset may only ever DELAY a probe, never bring it forward.
16215    ///
16216    /// A delay below the cadence would probe a module more often than
16217    /// configured, which is the opposite of what an operator asked for and
16218    /// would tighten the failure budget without anyone changing it.
16219    #[test]
16220    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16221        let cadence = Duration::from_secs(30);
16222        let span = cadence / 10;
16223        for id in FLEET {
16224            for probe_index in 0..8 {
16225                let delay = jittered_health_delay(id, probe_index, cadence);
16226                assert!(
16227                    delay >= cadence,
16228                    "{id}#{probe_index}: jitter must not shorten the cadence"
16229                );
16230                assert!(
16231                    delay < cadence + span,
16232                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16233                );
16234            }
16235        }
16236    }
16237
16238    /// A module keeps its offset across daemon restarts.
16239    ///
16240    /// The delay is derived rather than randomised precisely so a restart does
16241    /// not re-roll every module into a fresh chance of collision. A random
16242    /// source would satisfy the dispersal test above and quietly lose this.
16243    #[test]
16244    fn a_module_offset_is_stable_across_restarts() {
16245        let cadence = Duration::from_secs(30);
16246        for id in FLEET {
16247            assert_eq!(
16248                jittered_health_delay(id, 0, cadence),
16249                jittered_health_delay(id, 0, cadence),
16250                "{id}: the same module and probe index must produce the same offset"
16251            );
16252        }
16253    }
16254
16255    /// A zero cadence disables probing rather than producing a busy loop.
16256    #[test]
16257    fn zero_cadence_yields_zero_delay() {
16258        assert_eq!(
16259            jittered_health_delay("aft", 0, Duration::ZERO),
16260            Duration::ZERO
16261        );
16262    }
16263}
16264
16265#[cfg(all(test, target_os = "linux"))]
16266mod cgroup_placement_tests {
16267    use super::{
16268        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16269        SuperviseError, SupervisedChild,
16270    };
16271    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16272    use cortexkit_test_support::ScratchDir;
16273    use std::{
16274        fs, io,
16275        path::{Path, PathBuf},
16276        sync::{Arc, Mutex},
16277    };
16278    use tokio::process::Command;
16279
16280    #[tokio::test]
16281    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16282        use super::*;
16283        let dir = ScratchDir::new("unique-spawn-cgroups");
16284        let root = PathBuf::from(format!(
16285            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16286            std::process::id(),
16287            unix_ms_now()
16288        ));
16289        if let Err(error) = fs::create_dir(&root) {
16290            assert!(
16291                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16292                "required cgroup test cannot execute: {error}"
16293            );
16294            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16295            return;
16296        }
16297        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16298        let group_count = || {
16299            fs::read_dir(root.join("subc-modules"))
16300                .unwrap()
16301                .map(|entry| entry.unwrap().file_type().unwrap())
16302                .filter(|kind| kind.is_dir())
16303                .count()
16304        };
16305        let supervisor =
16306            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16307                .with_cgroup_placement(Some(placement.clone()));
16308        let runtime = supervisor.runtime_config();
16309        let mut spec = ModuleSpec {
16310            module_id: "unique-spawn".into(),
16311            program: PathBuf::from("/bin/sleep"),
16312            args: vec!["60".into()],
16313            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16314                .into_iter()
16315                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16316                .collect(),
16317            reserved: false,
16318            reserved_prefixes: vec![],
16319            protocol: ModuleProtocol::None,
16320            overlap: Default::default(),
16321        };
16322        let spawn = |spec: &ModuleSpec| {
16323            spawn_child(
16324                spec,
16325                None,
16326                None,
16327                &runtime.stderr_ring,
16328                None,
16329                &runtime.child_roster,
16330                Some(&placement),
16331            )
16332            .unwrap()
16333        };
16334        let mut live = spawn(&spec);
16335        for _ in 0..3 {
16336            // A new process can enter the old slot while retirement is pending.
16337            let next = spawn(&spec);
16338            assert_ne!(live.module_id, next.module_id);
16339            live.start_kill().unwrap();
16340            live.wait().await.unwrap();
16341            live = next;
16342            assert!(
16343                live.child.try_wait().unwrap().is_none(),
16344                "retiring the old slot must not kill the replacement"
16345            );
16346            assert_eq!(
16347                group_count(),
16348                1,
16349                "only the live spawn's cgroup should remain"
16350            );
16351        }
16352        supervisor.begin_daemon_shutdown();
16353        let reap = tokio::spawn(async move {
16354            live.wait().await.unwrap();
16355        });
16356        supervisor
16357            .end_children_for_daemon_shutdown(false, std::future::pending())
16358            .await;
16359        reap.await.unwrap();
16360        assert_eq!(group_count(), 0);
16361        // A normal exit uses the same tree-cleanup path as a killed spawn.
16362        spec.program = PathBuf::from("/bin/true");
16363        spec.args.clear();
16364        let fresh_roster = ChildRoster::default();
16365        let mut short = spawn_child(
16366            &spec,
16367            None,
16368            None,
16369            &runtime.stderr_ring,
16370            None,
16371            &fresh_roster,
16372            Some(&placement),
16373        )
16374        .unwrap();
16375        short.wait().await.unwrap();
16376        assert_eq!(group_count(), 0);
16377        spec.module_id = "_".repeat(255);
16378        let mut long_id = spawn_child(
16379            &spec,
16380            None,
16381            None,
16382            &runtime.stderr_ring,
16383            None,
16384            &fresh_roster,
16385            Some(&placement),
16386        )
16387        .unwrap();
16388        long_id.wait().await.unwrap();
16389        assert_eq!(
16390            group_count(),
16391            0,
16392            "valid long module IDs must not exceed cgroup NAME_MAX"
16393        );
16394        fs::remove_dir(root.join("subc-modules")).unwrap();
16395        fs::remove_dir(root).unwrap();
16396        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16397    }
16398
16399    #[test]
16400    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16401        let path = Path::new("/definitely-missing-subc-cgroup");
16402        let mut command = Command::new("true");
16403        let error = apply_cgroup_placement(
16404            &mut command,
16405            &ModuleSpec {
16406                module_id: "broken-cgroup".to_string(),
16407                program: PathBuf::from("true"),
16408                args: Vec::new(),
16409                env: Vec::new(),
16410                reserved: false,
16411                reserved_prefixes: Vec::new(),
16412                protocol: ModuleProtocol::Subc,
16413                overlap: Default::default(),
16414            },
16415            path,
16416        )
16417        .expect_err("a parent cgroup open failure must reject the supervised spawn");
16418        let reason = error.to_string();
16419
16420        assert!(
16421            matches!(error, SuperviseError::Cgroup { .. }),
16422            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16423        );
16424        assert!(
16425            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16426            "parent cgroup open failure must name cgroup.procs: {reason}"
16427        );
16428    }
16429
16430    #[tokio::test]
16431    async fn reaping_a_child_removes_its_empty_module_cgroup() {
16432        let root = ScratchDir::new("supervisor-reap-cgroup");
16433        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16434        let placement = subc_cgroup::prepare_at(&root)
16435            .expect("prepare scratch cgroup root")
16436            .expect("scratch root has a cgroup.procs marker");
16437        let module_id = "reaped-module";
16438        let module = placement
16439            .module_path(module_id)
16440            .expect("create scratch module cgroup");
16441        let child = Command::new("true")
16442            .env("XDG_DATA_HOME", root.path())
16443            .env("XDG_RUNTIME_DIR", root.path())
16444            .env("XDG_CONFIG_HOME", root.path())
16445            .spawn()
16446            .expect("spawn short-lived child");
16447        let pid = child.id().expect("spawned child has pid");
16448        let mut child = SupervisedChild {
16449            child,
16450            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16451            protocol: ModuleProtocol::Subc,
16452            module_id: module_id.to_string(),
16453            cgroup_placement: Some(placement),
16454            stdout_pump: None,
16455            stderr_pump: None,
16456            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16457            spawned_at: tokio::time::Instant::now(),
16458            registration_deadline: None,
16459            registration_watch: None,
16460            spawned_at_ms: 0,
16461            spawned_from: PathBuf::from("true"),
16462            launch_spec_digest: LaunchSpecDigest([0; 32]),
16463            spawned_file_identity: None,
16464            process_start_time: None,
16465            process_identity: None,
16466            pid,
16467            roster_guard: None,
16468            #[cfg(target_os = "macos")]
16469            privacy_exec: None,
16470            spawn_failure: None,
16471        };
16472
16473        child.wait().await.expect("reap short-lived child");
16474
16475        assert!(
16476            !module.exists(),
16477            "reaping the supervised child must remove its empty cgroup"
16478        );
16479    }
16480
16481    #[test]
16482    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16483        let root = ScratchDir::new("supervisor-non-empty-cgroup");
16484        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16485        let placement = subc_cgroup::prepare_at(&root)
16486            .expect("prepare scratch cgroup root")
16487            .expect("scratch root has a cgroup.procs marker");
16488        let module = placement
16489            .module_path("surviving-module")
16490            .expect("create scratch module cgroup");
16491        fs::write(module.join("surviving-process"), b"still present")
16492            .expect("make scratch cgroup non-empty");
16493        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16494
16495        remove_module_cgroup(&placement, "surviving-module");
16496
16497        let logs = crate::router::test_log::captured_logs(&logs);
16498        assert!(
16499            module.exists(),
16500            "failed removal must leave the cgroup intact"
16501        );
16502        assert!(
16503            logs.contains("could not remove module cgroup after process exit; continuing teardown")
16504                && logs.contains("surviving-module"),
16505            "best-effort removal must report the failure without returning it: {logs}"
16506        );
16507    }
16508
16509    #[test]
16510    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16511        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16512        let reason = SuperviseError::Spawn {
16513            program: PathBuf::from("/bin/true"),
16514            source: io::Error::from_raw_os_error(13),
16515            cgroup_path: Some(cgroup_path.clone()),
16516        }
16517        .to_string();
16518
16519        assert!(
16520            reason.contains(&cgroup_path.display().to_string()),
16521            "a pre_exec spawn failure must name the cgroup path: {reason}"
16522        );
16523    }
16524}
16525
16526#[cfg(test)]
16527mod spawn_subscriber_lag_tests {
16528    use super::*;
16529
16530    /// A subscriber whose connection stops draining is dropped once its frame
16531    /// channel fills. The client must learn that from a terminal Error frame
16532    /// after the frames already queued for it, not from a stream that simply
16533    /// goes quiet.
16534    #[tokio::test]
16535    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16536        let feed = SpawnEventFeed::default();
16537        feed.configure_incarnation("lag-incarnation".to_string());
16538        // A one-slot connection queue that nobody reads until the emits are
16539        // done: the forwarder parks on it and the subscriber channel fills.
16540        let (tx, mut rx) = mpsc::channel(1);
16541        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16542            .expect("subscribe");
16543        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16544        for index in 0..emitted {
16545            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16546            // Let the forwarder take what it can so the fill point is the
16547            // subscriber channel, not a scheduling accident.
16548            tokio::task::yield_now().await;
16549        }
16550        assert_eq!(
16551            feed.subscriber_count(),
16552            0,
16553            "the lagged subscriber must be removed"
16554        );
16555
16556        let mut data = Vec::new();
16557        let mut last = None;
16558        loop {
16559            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16560                .await
16561                .expect("the forwarder must finish once the subscriber is dropped");
16562            let Some(outbound) = next else { break };
16563            let frame = outbound.frame;
16564            if frame.header.ty == FrameType::StreamData {
16565                assert!(last.is_none(), "no data may follow the terminal frame");
16566                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16567                data.push(event.cursor.seq);
16568            } else {
16569                assert!(last.is_none(), "exactly one terminal frame");
16570                last = Some(frame);
16571            }
16572        }
16573        assert!(!data.is_empty(), "queued frames drain before the terminal");
16574        for pair in data.windows(2) {
16575            assert_eq!(
16576                pair[1],
16577                pair[0] + 1,
16578                "queued frames arrive dense and in order"
16579            );
16580        }
16581        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16582        assert_eq!(terminal.header.ty, FrameType::Error);
16583        assert_eq!(terminal.header.corr, 7);
16584        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16585        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16586        let detail = body.detail.expect("lagged error carries detail");
16587        assert_eq!(
16588            detail["first_undelivered_cursor"]["seq"],
16589            data.last().unwrap() + 1,
16590            "the named cursor is the first event the subscriber did not receive"
16591        );
16592        assert_eq!(
16593            detail["first_undelivered_cursor"]["daemon_incarnation"],
16594            "lag-incarnation"
16595        );
16596    }
16597}
16598
16599#[cfg(test)]
16600mod terminal_history_read_concurrency_tests {
16601    use super::*;
16602    use crate::terminal_journal::read_pause;
16603    use cortexkit_test_support::ScratchDir;
16604    use std::sync::mpsc as std_mpsc;
16605
16606    fn journaled_ring(
16607        journal: &Arc<crate::terminal_journal::TerminalJournal>,
16608    ) -> Arc<Mutex<TerminalRing>> {
16609        Arc::new(Mutex::new(
16610            TerminalRing::new(TerminalRingConfig::default(), 1)
16611                .with_journal(Some(Arc::clone(journal))),
16612        ))
16613    }
16614
16615    fn crash(at_ms: u64) -> ExitReport {
16616        ExitReport {
16617            kind: ExitKind::Crash,
16618            code: Some(1),
16619            signal: None,
16620            at_ms,
16621        }
16622    }
16623
16624    /// Record an exit on another thread and report whether it finished within
16625    /// `bound`. The recorder thread is left running if it did not.
16626    fn record_within(
16627        module_id: &'static str,
16628        ring: &Arc<Mutex<TerminalRing>>,
16629        at_ms: u64,
16630        bound: Duration,
16631    ) -> bool {
16632        let ring = Arc::clone(ring);
16633        let (done, done_rx) = std_mpsc::channel();
16634        std::thread::spawn(move || {
16635            record_terminal(
16636                module_id,
16637                &ring,
16638                &SpawnEventFeed::default(),
16639                &crash(at_ms),
16640                TerminalDisposition::Restarting,
16641            );
16642            let _ = done.send(());
16643        });
16644        done_rx.recv_timeout(bound).is_ok()
16645    }
16646
16647    /// A history read in progress must not hold the journal writer (which every
16648    /// module's exit recording needs) or the module's own ring. Exits recorded
16649    /// while the read is paused complete promptly; the paused read answers as of
16650    /// the moment it started, and the next read has each exit exactly once.
16651    #[test]
16652    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16653        let dir = ScratchDir::new("terminal-history-concurrent-read");
16654        let path = dir.join("terminals.jsonl");
16655        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16656            path.clone(),
16657            "daemon".into(),
16658        ));
16659        let reader_ring = journaled_ring(&journal);
16660        let other_ring = journaled_ring(&journal);
16661        assert!(record_within(
16662            "reader-module",
16663            &reader_ring,
16664            10,
16665            Duration::from_secs(5)
16666        ));
16667
16668        let (started, release) = read_pause::install(&path);
16669        let reading = {
16670            let ring = Arc::clone(&reader_ring);
16671            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16672        };
16673        started
16674            .recv_timeout(Duration::from_secs(5))
16675            .expect("the history read reached its pause");
16676
16677        let bound = Duration::from_secs(1);
16678        assert!(
16679            record_within("other-module", &other_ring, 20, bound),
16680            "another module's exit waited on a history read (journal writer held)"
16681        );
16682        assert!(
16683            record_within("reader-module", &reader_ring, 30, bound),
16684            "the read module's own exit waited on its history read (ring held)"
16685        );
16686
16687        drop(release);
16688        let paused = reading.join().unwrap();
16689        assert_eq!(
16690            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16691            vec![10],
16692            "an exit recorded after the read began lands in neither half of it"
16693        );
16694        assert_eq!(paused.journal_skipped_lines, 0);
16695        assert_eq!(paused.journal_read_errors, 0);
16696
16697        let after = durable_terminal_history_of(&reader_ring, "reader-module");
16698        assert_eq!(
16699            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16700            vec![10, 30],
16701            "the next read merges ring and journal with no duplicate"
16702        );
16703        assert_eq!(after.journal_skipped_lines, 0);
16704    }
16705}
16706
16707/// What a restart does with the exited process's output readers. These drive
16708/// the same settle functions the supervisor calls, with a reader the test
16709/// holds, so a reader that has not been scheduled by the bound is a controlled
16710/// input rather than something only a loaded machine produces.
16711#[cfg(test)]
16712mod stderr_settle_tests {
16713    use std::{
16714        future::Future,
16715        io,
16716        pin::Pin,
16717        sync::{Arc, Mutex},
16718        task::{Context, Poll},
16719        time::Duration,
16720    };
16721
16722    use tokio::{
16723        io::{AsyncRead, ReadBuf},
16724        sync::oneshot,
16725        time::Instant,
16726    };
16727
16728    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16729    use crate::stderr_tail::{
16730        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16731        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16732    };
16733
16734    const BOUND: Duration = Duration::from_millis(250);
16735
16736    /// Yields `before`, then stays pending until the gate is released, then
16737    /// yields `after` and reaches EOF. The bytes after the gate were written
16738    /// by a process that has already exited; only the reader is behind.
16739    struct HeldReader {
16740        before: Option<Vec<u8>>,
16741        gate: Option<oneshot::Receiver<()>>,
16742        after: io::Cursor<Vec<u8>>,
16743    }
16744
16745    impl AsyncRead for HeldReader {
16746        fn poll_read(
16747            mut self: Pin<&mut Self>,
16748            cx: &mut Context<'_>,
16749            buf: &mut ReadBuf<'_>,
16750        ) -> Poll<io::Result<()>> {
16751            if let Some(bytes) = self.before.take() {
16752                buf.put_slice(&bytes);
16753                return Poll::Ready(Ok(()));
16754            }
16755            if let Some(gate) = self.gate.as_mut() {
16756                match Pin::new(gate).poll(cx) {
16757                    Poll::Pending => return Poll::Pending,
16758                    Poll::Ready(_) => self.gate = None,
16759                }
16760            }
16761            Pin::new(&mut self.after).poll_read(cx, buf)
16762        }
16763    }
16764
16765    struct DiscardSink;
16766
16767    impl OutputSink for DiscardSink {
16768        fn write_line(&mut self, _line: &[u8]) {}
16769    }
16770
16771    fn line(text: &str) -> TailEntry {
16772        TailEntry::Line {
16773            text: text.to_string(),
16774            truncated: false,
16775            at_ms: None,
16776        }
16777    }
16778
16779    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16780        ring.lock().unwrap()
16781    }
16782
16783    /// Start a reader for a new process generation that delivers `before`
16784    /// immediately and `after` only once the returned sender fires (or is
16785    /// dropped).
16786    fn held_pump(
16787        ring: &Arc<Mutex<StderrRing>>,
16788        before: &str,
16789        after: &str,
16790    ) -> (StderrPump, oneshot::Sender<()>) {
16791        let generation = lock(ring).begin_process();
16792        let (release, gate) = oneshot::channel();
16793        let reader = HeldReader {
16794            before: Some(before.as_bytes().to_vec()),
16795            gate: Some(gate),
16796            after: io::Cursor::new(after.as_bytes().to_vec()),
16797        };
16798        let task = tokio::spawn(pump_stderr_to(
16799            reader,
16800            Arc::clone(ring),
16801            generation,
16802            DiscardSink,
16803        ));
16804        (StderrPump { task, generation }, release)
16805    }
16806
16807    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16808        for _ in 0..1000 {
16809            if done(&lock(ring)) {
16810                return;
16811            }
16812            tokio::time::sleep(Duration::from_millis(1)).await;
16813        }
16814        panic!(
16815            "ring never reached the expected state: {:?}",
16816            lock(ring).snapshot(None, None)
16817        );
16818    }
16819
16820    #[tokio::test(start_paused = true)]
16821    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16822        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16823        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16824
16825        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16826        let before_release = lock(&ring).snapshot(None, None);
16827        assert!(
16828            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16829            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16830        );
16831
16832        // The restart: the next process starts and writes before the old
16833        // reader catches up.
16834        let next = lock(&ring).begin_process();
16835        lock(&ring).push_line_from(next, "next process booting");
16836        release.send(()).unwrap();
16837        wait_until(&ring, |ring| {
16838            ring.snapshot(None, None).capture == CaptureState::Captured
16839        })
16840        .await;
16841
16842        assert_eq!(
16843            untimed(lock(&ring).snapshot(None, None).entries),
16844            vec![
16845                line("booting"),
16846                line("config error: missing storage"),
16847                TailEntry::ProcessStart,
16848                line("next process booting"),
16849            ],
16850            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16851        );
16852    }
16853
16854    #[tokio::test(start_paused = true)]
16855    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16856    ) {
16857        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16858        // `_held` is never fired: a descendant keeps the pipe open for the
16859        // whole test.
16860        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16861
16862        let started = Instant::now();
16863        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16864        assert_eq!(
16865            started.elapsed(),
16866            BOUND,
16867            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16868        );
16869
16870        let next = lock(&ring).begin_process();
16871        lock(&ring).push_line_from(next, "next process booting");
16872        tokio::time::sleep(Duration::from_secs(60)).await;
16873
16874        let snapshot = lock(&ring).snapshot(None, None);
16875        match &snapshot.capture {
16876            CaptureState::Incomplete { reason } => assert!(
16877                reason.contains("had not reached EOF") && reason.contains("250ms"),
16878                "the reason must say what is missing and after how long: {reason}"
16879            ),
16880            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16881        }
16882        assert_eq!(
16883            untimed(snapshot.entries),
16884            vec![
16885                line("parent exiting"),
16886                TailEntry::ProcessStart,
16887                line("next process booting"),
16888            ]
16889        );
16890    }
16891
16892    #[tokio::test(start_paused = true)]
16893    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16894        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16895        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16896        release.send(()).unwrap();
16897
16898        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16899
16900        let snapshot = lock(&ring).snapshot(None, None);
16901        assert_eq!(snapshot.capture, CaptureState::Captured);
16902        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16903    }
16904
16905    #[tokio::test(start_paused = true)]
16906    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16907        const BURST: usize = 150;
16908        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16909        let path = capture.join("two-pipe.stderr.log");
16910        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16911        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16912        let generation = lock(&ring).begin_process();
16913        let stdout = (0..BURST)
16914            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16915            .collect::<String>();
16916        let stderr = (0..BURST)
16917            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16918            .collect::<String>();
16919        // Model a child that has written both bursts and exited. Hold the stdout
16920        // reader after one line and a partial prefix, as if pipe readiness or
16921        // capture-file writes had delayed it. The final line also needs EOF to
16922        // flush: neither complete nor partial buffered output may be discarded.
16923        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16924        let (release, gate) = oneshot::channel();
16925        let reader = HeldReader {
16926            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16927            gate: Some(gate),
16928            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16929        };
16930        let (finished, completion) = oneshot::channel();
16931        let stdout_sink = sink.clone();
16932        let pump = tokio::spawn(async move {
16933            pump_stdout_to(reader, stdout_sink).await;
16934            finished.send(()).unwrap();
16935        });
16936        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16937
16938        let started = Instant::now();
16939        settle_stdout_pump("two-pipe", pump, BOUND).await;
16940        assert_eq!(
16941            started.elapsed(),
16942            BOUND,
16943            "a late reader must not hold up restart"
16944        );
16945        assert_eq!(
16946            std::fs::read_to_string(&path).unwrap().lines().count(),
16947            BURST + 1
16948        );
16949
16950        release
16951            .send(())
16952            .expect("stdout reader must survive the exit drain bound");
16953        tokio::time::timeout(Duration::from_secs(1), completion)
16954            .await
16955            .expect("released stdout reader must reach EOF")
16956            .expect("stdout capture must finish at EOF, not be cancelled");
16957        let contents = std::fs::read_to_string(&path).unwrap();
16958        let mut lines = contents
16959            .lines()
16960            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16961            .collect::<Vec<_>>();
16962        let mut expected = (0..BURST)
16963            .flat_map(|i| {
16964                [
16965                    format!("out-{i:04}-{}", "o".repeat(64)),
16966                    format!("err-{i:04}-{}", "e".repeat(64)),
16967                ]
16968            })
16969            .collect::<Vec<_>>();
16970        lines.sort_unstable();
16971        expected.sort_unstable();
16972        assert_eq!(
16973            lines, expected,
16974            "both pipes must retain every complete line exactly once"
16975        );
16976    }
16977}
16978
16979/// Containment of a module's process tree (issue #109).
16980///
16981/// The behaviour these defend against is a module helper surviving its module:
16982/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
16983/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
16984/// compounds it.
16985///
16986/// They run against the SUPERVISOR rather than the job-object crate because the
16987/// claim is about teardown: a crate-level test proves a job can reap a tree, not
16988/// that the daemon's drain path reaches it.
16989///
16990/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
16991/// lane there is a separate containment path with its own tests.
16992#[cfg(all(test, windows))]
16993mod job_containment_tests {
16994    use super::*;
16995    use cortexkit_test_support::ScratchDir;
16996    use std::{
16997        path::{Path, PathBuf},
16998        sync::{Arc, Mutex},
16999        time::{Duration, Instant},
17000    };
17001
17002    /// The stub, expected beside this test executable.
17003    ///
17004    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17005    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17006    /// failure then reads as a broken test rather than an unbuilt dependency.
17007    fn stub_path() -> PathBuf {
17008        let mut path = std::env::current_exe().expect("current_exe available in tests");
17009        path.pop();
17010        path.pop();
17011        path.push("fake-aft-stub.exe");
17012        assert!(
17013            path.exists(),
17014            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17015             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17016            path.display()
17017        );
17018        path
17019    }
17020
17021    /// Poll for the grandchild pid the stub records, and parse it.
17022    fn read_grandchild_pid(path: &Path) -> u32 {
17023        let deadline = Instant::now() + Duration::from_secs(10);
17024        loop {
17025            if let Ok(contents) = std::fs::read_to_string(path) {
17026                if let Ok(pid) = contents.trim().parse() {
17027                    return pid;
17028                }
17029            }
17030            assert!(
17031                Instant::now() < deadline,
17032                "the stub never recorded a grandchild pid at {}",
17033                path.display()
17034            );
17035            std::thread::sleep(Duration::from_millis(10));
17036        }
17037    }
17038
17039    /// Everything one fixture run needs, so the two tests below differ in exactly
17040    /// one place: whether the child is contained.
17041    struct Fixture {
17042        _dir: ScratchDir,
17043        module_id: String,
17044        grandchild: u32,
17045        child: Option<SupervisedChild>,
17046        registry: Arc<Registry>,
17047        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17048        terminal_ring: Arc<Mutex<TerminalRing>>,
17049        spawn_events: SpawnEventFeed,
17050    }
17051
17052    fn fixture(label: &str, module_id: &str) -> Fixture {
17053        let dir = ScratchDir::new(label);
17054        let pid_file = dir.join("grandchild.pid");
17055        let supervisor = Supervisor::new_for_test(
17056            Arc::new(Registry::default()),
17057            RestartPolicy::new(3, Duration::ZERO),
17058        );
17059        let runtime = supervisor.runtime_config();
17060        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17061        let spec = ModuleSpec {
17062            module_id: module_id.to_string(),
17063            program: stub_path(),
17064            // Zero args deliberately: a `--subc` argument would make the stub dial
17065            // a daemon that is not there, and the failure would land in the same
17066            // stderr ring this fixture exists to keep quiet.
17067            args: Vec::new(),
17068            env: vec![
17069                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17070                (
17071                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17072                    pid_file.display().to_string(),
17073                ),
17074            ],
17075            reserved: false,
17076            reserved_prefixes: Vec::new(),
17077            protocol: ModuleProtocol::Subc,
17078            overlap: Default::default(),
17079        };
17080        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17081            .expect("spawn the supervised fixture");
17082        let grandchild = read_grandchild_pid(&pid_file);
17083        Fixture {
17084            _dir: dir,
17085            module_id: module_id.to_string(),
17086            grandchild,
17087            child: Some(child),
17088            registry: Arc::new(Registry::default()),
17089            snapshot,
17090            terminal_ring: Arc::clone(&runtime.terminal_ring),
17091            spawn_events: SpawnEventFeed::default(),
17092        }
17093    }
17094
17095    impl Fixture {
17096        /// Drain through the supervisor's own teardown path.
17097        async fn drain(&mut self) {
17098            let child = self
17099                .child
17100                .take()
17101                .expect("the fixture child is still present");
17102            drain_child_to_state(
17103                &self.module_id,
17104                ModuleProtocol::Subc,
17105                // No forwarding table in this fixture, so nothing reaches the
17106                // child over a connection.
17107                StopNotice::NotSent,
17108                &self.registry,
17109                None,
17110                &self.snapshot,
17111                &self.terminal_ring,
17112                &self.spawn_events,
17113                child,
17114                Duration::from_millis(500),
17115                ModuleState::Stopped,
17116                Some(false),
17117                &FailureContext::operator(),
17118            )
17119            .await
17120            .expect("drain the supervised fixture");
17121        }
17122    }
17123
17124    /// Teardown reaps the grandchild, not merely the direct child.
17125    ///
17126    /// This is the assertion the change exists for. Before containment the
17127    /// grandchild survived: it is a separate process, and `start_kill` is
17128    /// `TerminateProcess` scoped to one pid.
17129    #[tokio::test]
17130    async fn teardown_reaps_the_grandchild() {
17131        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17132        let grandchild = fixture.grandchild;
17133
17134        assert!(
17135            subc_jobobject::process_exists(grandchild),
17136            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17137        );
17138
17139        fixture.drain().await;
17140
17141        assert!(
17142            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17143            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17144        );
17145    }
17146
17147    /// The mutation control: with containment withheld, the grandchild survives
17148    /// the same kill.
17149    ///
17150    /// This is the defect reproduction from #109 — a direct-child kill reaches
17151    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17152    /// supervisor because `spawn_and_mark_running` now always contains on
17153    /// Windows, which is the point: there is no longer a path that spawns
17154    /// uncontained, so the control has to construct one.
17155    ///
17156    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17157    /// grandchild ever dies here, that test is passing for a reason unrelated to
17158    /// the job object and the containment claim is unproven.
17159    #[test]
17160    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17161        let dir = ScratchDir::new("teardown-uncontained");
17162        let pid_file = dir.join("grandchild.pid");
17163        let mut child = std::process::Command::new(stub_path())
17164            .env("FAKE_AFT_NEVER_CONNECT", "1")
17165            .env(
17166                "FAKE_AFT_GRANDCHILD_PID_FILE",
17167                pid_file.display().to_string(),
17168            )
17169            .stdin(std::process::Stdio::null())
17170            .stdout(std::process::Stdio::null())
17171            .stderr(std::process::Stdio::null())
17172            .spawn()
17173            .expect("spawn the uncontained fixture");
17174        let grandchild = read_grandchild_pid(&pid_file);
17175
17176        // Exactly what the pre-fix teardown did: kill the direct child.
17177        child.kill().expect("kill the direct child");
17178        let _ = child.wait();
17179
17180        assert!(
17181            subc_jobobject::process_exists(grandchild),
17182            "grandchild {grandchild} died with the direct child, so this control no longer \
17183             distinguishes contained from uncontained teardown and the regression test is \
17184             passing vacuously"
17185        );
17186
17187        // The orphan this control demonstrates is the leak the fix prevents, so
17188        // the control must not leave one behind.
17189        kill_tree(grandchild);
17190    }
17191
17192    /// Crash durability: closing the containment handle reaps the tree with no
17193    /// teardown code running at all.
17194    ///
17195    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17196    /// call anything — and it is why containment is a kernel property of the
17197    /// handle rather than a step in the drain. Discovered by getting the
17198    /// mutation control wrong: clearing `job` to "disable" containment instead
17199    /// killed the tree, which is the guarantee, not a mistake.
17200    #[tokio::test]
17201    async fn dropping_containment_reaps_the_grandchild() {
17202        let mut fixture = fixture("drop-containment", "tree-drop");
17203        let grandchild = fixture.grandchild;
17204
17205        assert!(subc_jobobject::process_exists(grandchild));
17206
17207        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17208        fixture.child.as_mut().expect("child present").job = None;
17209
17210        assert!(
17211            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17212            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17213             crash would leave the tree behind"
17214        );
17215    }
17216
17217    /// Kill a pid and its tree, then confirm it is gone.
17218    fn kill_tree(pid: u32) {
17219        let _ = std::process::Command::new("taskkill.exe")
17220            .args(["/PID", &pid.to_string(), "/T", "/F"])
17221            .stdin(std::process::Stdio::null())
17222            .stdout(std::process::Stdio::null())
17223            .stderr(std::process::Stdio::null())
17224            .status();
17225        assert!(
17226            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17227            "could not clean up grandchild {pid}"
17228        );
17229    }
17230}
17231
17232#[cfg(test)]
17233mod privacy_trampoline_configuration_tests {
17234    #[tokio::test]
17235    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17236    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17237        #[cfg(target_os = "macos")]
17238        {
17239            let supervisor = super::Supervisor::new(
17240                std::sync::Arc::new(crate::Registry::default()),
17241                super::RestartPolicy::default(),
17242            );
17243            let error = supervisor.spawn(spec()).unwrap_err();
17244            assert!(
17245                error
17246                    .to_string()
17247                    .contains("no privacy trampoline configured"),
17248                "{error}"
17249            );
17250        }
17251    }
17252
17253    #[tokio::test]
17254    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17255    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17256        #[cfg(target_os = "macos")]
17257        {
17258            let supervisor = super::Supervisor::new(
17259                std::sync::Arc::new(crate::Registry::default()),
17260                super::RestartPolicy::default(),
17261            )
17262            .with_privacy_trampoline(std::env::current_exe().unwrap());
17263            let error = supervisor.spawn(spec()).unwrap_err();
17264            assert!(
17265                error
17266                    .to_string()
17267                    .contains("binary does not implement the privacy trampoline protocol"),
17268                "{error}"
17269            );
17270        }
17271    }
17272
17273    #[cfg(target_os = "macos")]
17274    fn spec() -> super::ModuleSpec {
17275        super::ModuleSpec {
17276            module_id: "privacy-configuration".into(),
17277            program: "/bin/sleep".into(),
17278            args: vec!["30".into()],
17279            env: vec![],
17280            reserved: false,
17281            reserved_prefixes: vec![],
17282            protocol: subc_control::ModuleProtocol::None,
17283            overlap: super::ModuleOverlap::Exclusive,
17284        }
17285    }
17286}
17287
17288#[cfg(test)]
17289mod privacy_exec_boundary_tests {
17290    #[cfg(target_os = "macos")]
17291    use super::*;
17292    #[cfg(target_os = "macos")]
17293    use std::{
17294        io::{Read, Write},
17295        net::{TcpListener, TcpStream},
17296    };
17297
17298    /// Unit-test-only pause at the actual early image read, not at a later
17299    /// status read. Production supervisors never inspect this environment key.
17300    #[cfg(target_os = "macos")]
17301    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17302        if let Some((_, path)) = spec
17303            .env
17304            .iter()
17305            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17306        {
17307            let mut barrier = TcpStream::connect(path).unwrap();
17308            barrier
17309                .set_read_timeout(Some(Duration::from_secs(30)))
17310                .unwrap();
17311            barrier.write_all(&pid.to_ne_bytes()).unwrap();
17312            let mut release = [0];
17313            barrier.read_exact(&mut release).unwrap();
17314            assert_eq!(&release, b"X");
17315        }
17316    }
17317
17318    #[cfg(target_os = "macos")]
17319    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17320        ModuleSpec {
17321            module_id: "privacy-boundary".into(),
17322            program: program.into(),
17323            args: args.iter().map(|arg| (*arg).into()).collect(),
17324            env: vec![],
17325            reserved: false,
17326            reserved_prefixes: vec![],
17327            protocol: ModuleProtocol::None,
17328            overlap: ModuleOverlap::Exclusive,
17329        }
17330    }
17331
17332    #[cfg(target_os = "macos")]
17333    async fn accept(listener: TcpListener) -> TcpStream {
17334        // Socket readiness, not elapsed time, establishes both pause points.
17335        let listener = tokio::net::TcpListener::from_std({
17336            listener.set_nonblocking(true).unwrap();
17337            listener
17338        })
17339        .unwrap();
17340        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17341            .await
17342            .unwrap()
17343            .unwrap();
17344        let stream = stream.into_std().unwrap();
17345        stream.set_nonblocking(false).unwrap();
17346        stream
17347            .set_read_timeout(Some(Duration::from_secs(30)))
17348            .unwrap();
17349        stream
17350    }
17351
17352    #[tokio::test]
17353    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17354    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17355        #[cfg(target_os = "macos")]
17356        {
17357            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17358            // Loopback sockets also work when the replay adapter's TMPDIR is
17359            // longer than Darwin's Unix-domain socket path limit.
17360            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17361            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17362            let record = root.join("live-children.json");
17363            let supervisor =
17364                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17365                    .with_live_children_record(&record);
17366            let runtime = supervisor.runtime_config();
17367            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17368            let mut spec = spec("/bin/sleep", &["30"]);
17369            spec.env = vec![
17370                (
17371                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17372                    exec_listener.local_addr().unwrap().to_string(),
17373                ),
17374                (
17375                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17376                    sample_listener.local_addr().unwrap().to_string(),
17377                ),
17378            ];
17379            let spawn = tokio::task::spawn_blocking(move || {
17380                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17381            });
17382            let mut sample = accept(sample_listener).await;
17383            let mut pid = [0; 4];
17384            sample.read_exact(&mut pid).unwrap();
17385            let pid = u32::from_ne_bytes(pid);
17386            let mut exec = accept(exec_listener).await;
17387            let mut ready = [0];
17388            exec.read_exact(&mut ready).unwrap();
17389            assert_eq!(&ready, b"R");
17390            // The early read is guaranteed to see a real, non-null trampoline
17391            // image: the fixture has reached its barrier and cannot exec yet.
17392            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17393            assert_eq!(
17394                observe_spawned_image(pid).unwrap().executable,
17395                Some(trampoline)
17396            );
17397            sample.write_all(b"X").unwrap();
17398            let mut child = spawn.await.unwrap();
17399            let early = crate::live_children::read_record(&record).unwrap();
17400            assert_eq!(early.len(), 1);
17401            assert_eq!(early[0].pid, pid);
17402            assert_eq!(
17403                early[0].executable, None,
17404                "unconfirmed trampoline image entered the roster"
17405            );
17406            assert!(child.report_ready.get().is_none());
17407            // The barrier's duration is unrelated to the production five-second
17408            // exec budget. Start the test's confirmation budget upon release.
17409            child.privacy_exec.as_mut().unwrap().deadline =
17410                tokio::time::Instant::now() + Duration::from_secs(30);
17411            exec.write_all(b"X").unwrap();
17412            child.confirm_privacy_exec().await;
17413            assert_eq!(child.spawn_failure, None);
17414            assert!(child.report_ready.get().is_some());
17415            let confirmed = crate::live_children::read_record(&record).unwrap();
17416            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17417            assert_ne!(module, trampoline);
17418            assert_eq!(confirmed[0].executable, Some(module.into()));
17419            child.start_kill().unwrap();
17420            child.wait().await.unwrap();
17421            child.release_roster();
17422        }
17423    }
17424
17425    #[tokio::test]
17426    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17427    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17428        #[cfg(target_os = "macos")]
17429        {
17430            let registry = Arc::new(Registry::default());
17431            let policy = RestartPolicy::new(0, Duration::ZERO);
17432            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
17433            let runtime = supervisor.runtime_config();
17434            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17435            let spec = spec("/bin/sh", &["-c", "exit 121"]);
17436            // Drive spawn and confirmation separately instead of starting the
17437            // monitor. WNOWAIT observes a real exit without consuming its status,
17438            // so confirmation's first try_wait must take the already-exited arm.
17439            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17440            let pid = child.pid;
17441            tokio::task::spawn_blocking(move || {
17442                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17443            })
17444            .await
17445            .unwrap()
17446            .unwrap();
17447            child.privacy_exec.as_mut().unwrap().deadline =
17448                tokio::time::Instant::now() + Duration::from_secs(30);
17449            let status = child.wait().await.unwrap();
17450            assert_eq!(status.code(), Some(121));
17451            assert!(child.privacy_exec.is_none());
17452            assert!(
17453                child.report_ready.get().is_none(),
17454                "an exited module must not publish a live pid"
17455            );
17456            let report = classify_reaped_child_exit(&snapshot, &child, &status);
17457            on_child_exit(
17458                &spec,
17459                policy,
17460                &registry,
17461                &snapshot,
17462                &runtime.terminal_ring,
17463                &runtime.spawn_events,
17464                &runtime.child_roster,
17465                report,
17466                true,
17467            )
17468            .await;
17469            let state = lock_snapshot(&snapshot).unwrap();
17470            assert_eq!(state.state, ModuleState::Failed);
17471            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17472            assert_eq!(state.reported_pid(), None);
17473            drop(state);
17474            let history = runtime.terminal_ring.lock().unwrap().snapshot();
17475            assert_eq!(history.entries.len(), 1);
17476            let terminal = &history.entries[0];
17477            assert_eq!(terminal.exit_code, Some(121));
17478            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17479            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17480            assert_eq!(
17481                terminal.disposition_detail.as_deref(),
17482                Some(policy.budget_exhausted_detail().as_str()),
17483                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17484            );
17485            assert_eq!(child.spawn_failure, None);
17486            child.release_roster();
17487        }
17488    }
17489}
17490
17491/// The daemon's real spawn path hands a subc-wire child its launch nonce on
17492/// descriptor 3, without an environment copy. The shell records the nonce
17493/// and its environment after exec so these tests observe the real handover.
17494#[cfg(all(test, unix))]
17495mod launch_nonce_descriptor_tests {
17496    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17497    use crate::stderr_tail::{StderrRing, StderrTailConfig};
17498    use cortexkit_test_support::ScratchDir;
17499    use std::{
17500        path::PathBuf,
17501        sync::{Arc, Mutex},
17502        time::{Duration, Instant},
17503    };
17504
17505    async fn probe(role: super::SpawnRole) {
17506        let scratch = ScratchDir::new("launch-nonce-descriptor");
17507        let fd_copy = scratch.join("from-descriptor");
17508        let env_copy = scratch.join("environment");
17509        let script = format!(
17510            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17511            fd = fd_copy.display(), env = env_copy.display(),
17512        );
17513        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17514        let spec = ModuleSpec {
17515            module_id: "nonce-descriptor-probe".to_string(),
17516            program: PathBuf::from("/bin/sh"),
17517            args: vec!["-c".to_string(), script],
17518            env: vec![
17519                xdg("XDG_DATA_HOME"),
17520                xdg("XDG_RUNTIME_DIR"),
17521                xdg("XDG_CONFIG_HOME"),
17522                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17523            ],
17524            reserved: true,
17525            reserved_prefixes: Vec::new(),
17526            protocol: ModuleProtocol::Subc,
17527            overlap: Default::default(),
17528        };
17529        let handle = SupervisorHandle::new();
17530        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17531        let roster = ChildRoster::default();
17532        #[cfg(target_os = "macos")]
17533        {
17534            let path = super::test_privacy_trampoline();
17535            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17536        }
17537        let child = super::spawn_child_in_slot(
17538            &spec,
17539            None,
17540            Some(&handle),
17541            &ring,
17542            None,
17543            &roster,
17544            #[cfg(target_os = "linux")]
17545            None,
17546            role,
17547            matches!(role, super::SpawnRole::SwapCandidate),
17548        )
17549        .expect("spawn probe");
17550        let deadline = Instant::now() + Duration::from_secs(10);
17551        while !(fd_copy.exists() && env_copy.exists()) {
17552            assert!(Instant::now() < deadline, "probe never wrote its copies");
17553            tokio::time::sleep(Duration::from_millis(20)).await;
17554        }
17555        let nonce = std::fs::read_to_string(fd_copy).unwrap();
17556        assert!(!nonce.is_empty());
17557        let environment = std::fs::read_to_string(env_copy).unwrap();
17558        assert!(environment
17559            .lines()
17560            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17561        let copy = environment
17562            .lines()
17563            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17564        assert_eq!(
17565            copy, None,
17566            "Unix children must never receive the environment nonce"
17567        );
17568        if matches!(role, super::SpawnRole::Plain) {
17569            assert_eq!(
17570                handle.spawn_nonce(&spec.module_id).as_deref(),
17571                Some(nonce.as_str())
17572            );
17573        }
17574        drop(child);
17575    }
17576
17577    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17578    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17579        probe(super::SpawnRole::Plain).await;
17580    }
17581
17582    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17583    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17584        probe(super::SpawnRole::SwapCandidate).await;
17585    }
17586}
17587
17588#[cfg(all(test, target_os = "linux"))]
17589mod cgroup_containment_tests {
17590    use super::*;
17591    use cortexkit_test_support::ScratchDir;
17592
17593    fn running(pid: u32) -> bool {
17594        // An orphan can remain a zombie until the container init reaps it.
17595        std::fs::read_to_string(format!("/proc/{pid}/stat"))
17596            .ok()
17597            .and_then(|stat| {
17598                stat.rsplit_once(") ")
17599                    .map(|(_, rest)| rest.starts_with('Z'))
17600            })
17601            .is_some_and(|zombie| !zombie)
17602    }
17603
17604    #[tokio::test]
17605    async fn linux_teardown_reaps_the_grandchild() {
17606        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17607    }
17608
17609    #[tokio::test]
17610    async fn linux_shutdown_straggler_reaps_the_grandchild() {
17611        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17612    }
17613
17614    async fn teardown_tree(test_name: &str, shutdown: bool) {
17615        let dir = ScratchDir::new(test_name);
17616        let root = PathBuf::from(format!(
17617            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17618            std::process::id(),
17619            unix_ms_now()
17620        ));
17621        if let Err(error) = std::fs::create_dir(&root) {
17622            assert!(
17623                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17624                "required cgroup test cannot execute: {error}"
17625            );
17626            eprintln!(
17627                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17628                root.display()
17629            );
17630            return;
17631        }
17632        let placement = subc_cgroup::prepare_at(&root)
17633            .expect("prepare isolated kernel cgroup")
17634            .expect("isolated cgroup is delegated");
17635        let module_id = "tree-teardown";
17636        let module = placement
17637            .module_path(module_id)
17638            .expect("create isolated module cgroup");
17639        if !module.join("cgroup.kill").exists() {
17640            std::fs::remove_dir(&module).unwrap();
17641            std::fs::remove_dir(root.join("subc-modules")).unwrap();
17642            std::fs::remove_dir(&root).unwrap();
17643            assert!(
17644                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17645                "required cgroup.kill interface unavailable"
17646            );
17647            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17648            return;
17649        }
17650        let supervisor = Supervisor::new_for_test(
17651            Arc::new(Registry::default()),
17652            RestartPolicy::new(3, Duration::ZERO),
17653        )
17654        .with_cgroup_placement(Some(placement));
17655        let mut runtime = supervisor.runtime_config();
17656        runtime.child_roster = runtime
17657            .child_roster
17658            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17659        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17660        let pid_file = dir.join("grandchild.pid");
17661        let spec = ModuleSpec {
17662            module_id: module_id.to_string(),
17663            program: PathBuf::from("/bin/sh"),
17664            args: vec![
17665                "-c".into(),
17666                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17667                "fixture".into(),
17668                pid_file.display().to_string(),
17669            ],
17670            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17671                .into_iter()
17672                .map(|key| (key.to_string(), dir.display().to_string()))
17673                .collect(),
17674            reserved: false,
17675            reserved_prefixes: Vec::new(),
17676            protocol: ModuleProtocol::None,
17677            overlap: Default::default(),
17678        };
17679        let child =
17680            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17681        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17682        let grandchild: u32 = loop {
17683            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17684                if let Ok(pid) = contents.trim().parse() {
17685                    break pid;
17686                }
17687            }
17688            assert!(
17689                tokio::time::Instant::now() < deadline,
17690                "grandchild pid was not recorded"
17691            );
17692            tokio::time::sleep(Duration::from_millis(10)).await;
17693        };
17694        assert!(
17695            running(grandchild),
17696            "grandchild must be alive before teardown"
17697        );
17698        if shutdown {
17699            let mut child = child;
17700            crate::child_roster::end_children_for_daemon_shutdown(
17701                &runtime.child_roster,
17702                false,
17703                std::future::pending(),
17704            )
17705            .await;
17706            child.wait().await.expect("reap shutdown straggler");
17707        } else {
17708            drain_child_to_state(
17709                module_id,
17710                ModuleProtocol::None,
17711                StopNotice::NotSent,
17712                &Registry::default(),
17713                None,
17714                &snapshot,
17715                &runtime.terminal_ring,
17716                &SpawnEventFeed::default(),
17717                child,
17718                Duration::from_millis(100),
17719                ModuleState::Stopped,
17720                Some(false),
17721                &FailureContext::operator(),
17722            )
17723            .await
17724            .expect("real supervisor teardown");
17725        }
17726        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17727        while running(grandchild) && tokio::time::Instant::now() < deadline {
17728            tokio::time::sleep(Duration::from_millis(10)).await;
17729        }
17730        let survived = running(grandchild);
17731        // Kill a surviving grandchild so a failed test does not leave it behind.
17732        if survived {
17733            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17734            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17735            tokio::time::sleep(Duration::from_millis(100)).await;
17736        }
17737        if module.exists() {
17738            std::fs::remove_dir(&module).expect("remove empty module cgroup");
17739        }
17740        std::fs::remove_dir(root.join("subc-modules")).unwrap();
17741        std::fs::remove_dir(&root).unwrap();
17742        assert!(
17743            !survived,
17744            "grandchild {grandchild} outlived module teardown"
17745        );
17746        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17747    }
17748}
17749
17750/// A slow live-children record write on the restart path must not stop the
17751/// daemon from answering the replacement module's authentication handshake.
17752#[cfg(all(test, unix))]
17753mod record_write_stall_tests {
17754    use super::*;
17755    use std::sync::atomic::{AtomicBool, Ordering};
17756    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17757
17758    /// `subc-client-rs` gives the whole handshake this long before the module
17759    /// exits with `Timeout { stage: ServerProof }`.
17760    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17761    /// How long the replacement's admission write is held: longer than the
17762    /// module's deadline, standing in for the delay a busy disk can add
17763    /// across the two syncs in that write.
17764    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17765
17766    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
17767    /// On macOS every module is launched through the privacy trampoline, and
17768    /// a module that is not a `#!` script is admitted only if its running
17769    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
17770    /// the shell soon replaces itself with `/bin/sleep`, which does not match
17771    /// the configured `/bin/sh`. The image is read only after the spawn's
17772    /// live-children record write, and here that write lasted long enough for
17773    /// the shell to have exec'd every time, so each process was refused and
17774    /// killed right after spawn, and the supervisor crash-looped it with
17775    /// backoff until its restart budget ran out. The restart this test
17776    /// measures would then never produce a module at all.
17777    fn sleeping_spec() -> ModuleSpec {
17778        ModuleSpec {
17779            module_id: "record-write-stall".into(),
17780            program: "/bin/sleep".into(),
17781            args: vec!["600".into()],
17782            env: vec![],
17783            reserved: false,
17784            reserved_prefixes: vec![],
17785            protocol: ModuleProtocol::None,
17786            overlap: Default::default(),
17787        }
17788    }
17789
17790    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17791    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17792        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17793        let supervisor =
17794            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17795                .with_live_children_record(home.join("live-children.json"));
17796        let module = supervisor.spawn(sleeping_spec()).unwrap();
17797        // `reported_pid` names a process only once it is confirmed as the
17798        // module (on macOS, once the trampoline's image check has passed), so
17799        // a process the daemon is about to refuse and kill never counts as
17800        // the running module, here or in the final wait below.
17801        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17802            loop {
17803                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17804                    .unwrap()
17805                    .reported_pid()
17806                {
17807                    return pid;
17808                }
17809                tokio::time::sleep(Duration::from_millis(10)).await;
17810            }
17811        })
17812        .await
17813        .expect("the first process was admitted as the module");
17814
17815        // The daemon's listener, served by this runtime's workers like the
17816        // supervisor's own tasks.
17817        let key = vec![0x42; 32];
17818        let daemon_id = [0x24; 16];
17819        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17820        let port = listener.local_addr().unwrap().port();
17821        let server = tokio::spawn(crate::server::serve_listener(
17822            listener,
17823            Arc::new(crate::router::Router::with_default_self_handler()),
17824            crate::server::ServerAuth::with_limits(
17825                key.clone(),
17826                daemon_id,
17827                "test",
17828                crate::server::DEFAULT_AUTH_DEADLINE,
17829                4,
17830            ),
17831        ));
17832
17833        // Hold the write that admits the replacement, once, and say when it
17834        // has started.
17835        let (held_tx, held_rx) = std::sync::mpsc::channel();
17836        let armed = AtomicBool::new(true);
17837        supervisor
17838            .child_roster
17839            .set_record_write_hook_for_test(move |live| {
17840                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17841                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17842                    let _ = held_tx.send(());
17843                    std::thread::sleep(HELD_RECORD_WRITE);
17844                }
17845            });
17846
17847        // The replacement module, as a separate process would be: its own
17848        // thread and runtime, so only the daemon's runtime can be stalled.
17849        let module_process = std::thread::spawn(move || {
17850            held_rx
17851                .recv_timeout(Duration::from_secs(30))
17852                .expect("the restart reached the replacement's record write");
17853            let conn = ConnectionInfo {
17854                schema: SCHEMA_VERSION,
17855                wire_version: None,
17856                endpoints: vec![Endpoint {
17857                    host: "127.0.0.1".to_owned(),
17858                    port,
17859                }],
17860                key,
17861                daemon_id,
17862                pid: std::process::id(),
17863                daemon_ver: "test".to_owned(),
17864            };
17865            tokio::runtime::Builder::new_current_thread()
17866                .enable_all()
17867                .build()
17868                .unwrap()
17869                .block_on(async {
17870                    let started = Instant::now();
17871                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17872                        .await
17873                        .unwrap();
17874                    let result =
17875                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17876                    (result, started.elapsed())
17877                })
17878        });
17879
17880        module.restart(Some(0)).await.unwrap();
17881        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17882            .await
17883            .unwrap();
17884        assert!(
17885            auth.is_ok(),
17886            "the daemon must answer a module's handshake while a restart waits on the \
17887             live-children record write; got {auth:?} after {elapsed:?}"
17888        );
17889
17890        tokio::time::timeout(Duration::from_secs(20), async {
17891            while lock_snapshot(&module.inner.snapshot)
17892                .unwrap()
17893                .reported_pid()
17894                .is_none_or(|pid| pid == old_pid)
17895            {
17896                tokio::time::sleep(Duration::from_millis(10)).await;
17897            }
17898        })
17899        .await
17900        .expect("the restart published its replacement");
17901        module.stop().await.unwrap();
17902        server.abort();
17903    }
17904}